rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity

Format the Rust code with rustfmt

Browse Source 
The code was formatted by hand, and `cargo fmt --check` differed in 866
places. rustfmt.toml keeps the house's long lines (120) and short
expressions on one line, which halves what changes; this commit is only
what rustfmt does with it, and is named in .git-blame-ignore-revs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-17 21:55:41 +00:00
parent 2c4ba7896e
commit 692a510217
26 changed files with 1710 additions and 1114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
crates/directory/src/apple.rs
View File

@@ -96,8 +96,11 @@ async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<(
}
let result = dir.store.snapshot(INSTANCE, &entries).await?;
tracing::info!(
shows = ids.len(), accepted = result.accepted, refused_private = result.refused_private,
refused_blocked = result.refused_blocked, refused_invalid = result.refused_invalid,
shows = ids.len(),
accepted = result.accepted,
refused_private = result.refused_private,
refused_blocked = result.refused_blocked,
refused_invalid = result.refused_invalid,
"Apple Podcasts synced"
);
// Fetched now, not when the refresher next looks.
@@ -128,7 +131,11 @@ struct Genre {
fn categories(g: &Genres) -> BTreeMap<String, Vec<String>> {
g.0.get("26")
.map(|podcasts| {
podcasts.subgenres.values().map(|c| (c.name.clone(), c.subgenres.values().map(|s| s.name.clone()).collect())).collect()
podcasts
.subgenres
.values()
.map(|c| (c.name.clone(), c.subgenres.values().map(|s| s.name.clone()).collect()))
.collect()
})
.unwrap_or_default()
}
@@ -253,7 +260,8 @@ mod tests {
)
.unwrap();
assert_eq!(chart_ids(&chart), ["1200361736", "1028908750"]);
let one: Chart = serde_json::from_str(r#"{"feed": {"entry": {"id": {"attributes": {"im:id": "7"}}}}}"#).unwrap();
let one: Chart =
serde_json::from_str(r#"{"feed": {"entry": {"id": {"attributes": {"im:id": "7"}}}}}"#).unwrap();
assert_eq!(chart_ids(&one), ["7"], "a chart of one");
let empty: Chart = serde_json::from_str(r#"{"feed": {}}"#).unwrap();
assert!(chart_ids(&empty).is_empty());

9
crates/directory/src/config.rs
View File

@@ -162,7 +162,10 @@ mod tests {
let cfg = Config::parse(GOOD).unwrap();
assert_eq!(cfg.bind, "0.0.0.0:8100");
let t = cfg.tokens().unwrap();
assert_eq!((t[0].name.as_str(), t[0].access, t[0].sha256.as_str()), ("tower", Access::Rwa, "a".repeat(64).as_str()));
assert_eq!(
(t[0].name.as_str(), t[0].access, t[0].sha256.as_str()),
("tower", Access::Rwa, "a".repeat(64).as_str())
);
assert_eq!(t[1].access, Access::Ro);
assert!(Access::Ro < Access::Rw && Access::Rw < Access::Rwa);
}
@@ -171,7 +174,9 @@ mod tests {
fn a_missing_or_misspelled_access_a_reused_name_or_a_bad_hash_is_refused() {
let h = "a".repeat(64);
assert!(Config::parse(&format!("[[tokens]]\nname = \"a\"\ntoken_sha256 = \"{h}\"")).is_err());
assert!(Config::parse(&format!("[[tokens]]\nname = \"a\"\naccess = \"admin\"\ntoken_sha256 = \"{h}\"")).is_err());
assert!(
Config::parse(&format!("[[tokens]]\nname = \"a\"\naccess = \"admin\"\ntoken_sha256 = \"{h}\"")).is_err()
);
assert!(Config::parse(&GOOD.replace("\"friend\"", "\"tower\"")).is_err());
assert!(Config::parse(&GOOD.replace(&"b".repeat(64), "xyz")).unwrap().tokens().is_err());
assert!(Config::parse(&GOOD.replace(&"b".repeat(64), &h)).unwrap().tokens().is_err());

83
crates/directory/src/lib.rs
View File

@@ -153,16 +153,16 @@ impl Directory {
fn caller(&self, presented: &str) -> Option<Me> {
let h = config::hash(presented);
// Every token compared, whether or not one matched, so timing says nothing either way.
self.tokens
.read()
.unwrap()
.iter()
.fold(None, |found, t| if config::same_secret(&t.sha256, &h) { Some(Me { name: t.name.clone(), access: t.access }) } else { found })
self.tokens.read().unwrap().iter().fold(None, |found, t| {
if config::same_secret(&t.sha256, &h) { Some(Me { name: t.name.clone(), access: t.access }) } else { found }
})
}
/// Fetches one feed and stores what it found.
pub async fn refresh(&self, due: &store::Due) -> anyhow::Result<()> {
let got = match ipx_feed::fetch(&self.client, &due.url, None, due.etag.as_deref(), due.last_modified.as_deref()).await {
let got = match ipx_feed::fetch(&self.client, &due.url, None, due.etag.as_deref(), due.last_modified.as_deref())
.await
{
Ok((ipx_feed::Fetched::NotModified, _)) => Refreshed::NotModified,
Ok((ipx_feed::Fetched::Body { bytes, etag, last_modified }, _)) => {
if ipx_feed::is_opml(&bytes) {
@@ -242,10 +242,14 @@ pub async fn reloader(dir: Shared, path: std::path::PathBuf) {
let n = tokens.len();
match dir.set_tokens(tokens).await {
Ok(dropped) => tracing::info!(tokens = n, dropped, "config read again"),
Err(e) => tracing::warn!(error = %format!("{e:#}"), "config read again, but old snapshots could not be dropped"),
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "config read again, but old snapshots could not be dropped")
}
}
}
Err(e) => tracing::warn!(error = %format!("{e:#}"), "config changed but does not load; keeping the tokens as they were"),
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "config changed but does not load; keeping the tokens as they were")
}
}
}
}
@@ -342,7 +346,8 @@ async fn authenticate(State(dir): State<Shared>, mut req: Request, next: Next) -
.map(str::trim)
.filter(|t| !t.is_empty());
let Some(me) = presented.and_then(|t| dir.caller(t)) else {
return ApiError(StatusCode::UNAUTHORIZED, "a valid token is needed: Authorization: Bearer <token>".into()).into_response();
return ApiError(StatusCode::UNAUTHORIZED, "a valid token is needed: Authorization: Bearer <token>".into())
.into_response();
};
let who = Who(me.name.clone());
req.extensions_mut().insert(me);
@@ -387,7 +392,11 @@ async fn popular(State(dir): State<Shared>, Query(q): Query<Limit>) -> Result<Js
}
async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
dir.store.feed(&id).await?.map(Json).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))
dir.store
.feed(&id)
.await?
.map(Json)
.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))
}
async fn snapshot(
@@ -408,7 +417,11 @@ async fn snapshot(
Ok(Json(r))
}
async fn remove(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
async fn remove(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(id): Path<String>,
) -> Result<StatusCode, ApiError> {
need(&me, Access::Rwa)?;
dir.store.block(&id).await?;
tracing::info!(feed = %id, by = %me.name, "feed removed and blocked");
@@ -420,7 +433,11 @@ async fn blocked(State(dir): State<Shared>, Extension(me): Extension<Me>) -> Res
Ok(Json(dir.store.blocked().await?))
}
async fn unblock(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
async fn unblock(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(id): Path<String>,
) -> Result<StatusCode, ApiError> {
need(&me, Access::Rwa)?;
if !dir.store.unblock(&id).await? {
return Err(ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not blocked")));
@@ -434,16 +451,28 @@ async fn instances(State(dir): State<Shared>, Extension(me): Extension<Me>) -> R
Ok(Json(dir.store.instances().await?))
}
async fn drop_instance(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(name): Path<String>) -> Result<StatusCode, ApiError> {
async fn drop_instance(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(name): Path<String>,
) -> Result<StatusCode, ApiError> {
need(&me, Access::Rwa)?;
let n = dir.store.drop_instance(&name).await?;
tracing::info!(instance = %name, feeds = n, by = %me.name, "snapshot dropped");
Ok(StatusCode::NO_CONTENT)
}
async fn refresh(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
async fn refresh(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(id): Path<String>,
) -> Result<Json<FeedPage>, ApiError> {
need(&me, Access::Rwa)?;
let due = dir.store.due_one(&id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))?;
let due = dir
.store
.due_one(&id)
.await?
.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))?;
dir.refresh(&due).await?;
feed(State(dir), Path(id)).await
}
@@ -467,7 +496,13 @@ mod tests {
(router(dir.clone()), dir)
}
async fn call(app: &Router, method: &str, path: &str, token: Option<&str>, body: Option<serde_json::Value>) -> (u16, serde_json::Value) {
async fn call(
app: &Router,
method: &str,
path: &str,
token: Option<&str>,
body: Option<serde_json::Value>,
) -> (u16, serde_json::Value) {
let mut req = axum::http::Request::builder().method(method).uri(path);
if let Some(t) = token {
req = req.header("authorization", format!("Bearer secret-{t}"));
@@ -501,7 +536,8 @@ mod tests {
.unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), 200);
let got: serde_json::Value = serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap();
let got: serde_json::Value =
serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap();
assert_eq!(got["accepted"], 1);
}
@@ -510,11 +546,8 @@ mod tests {
let (app, _) = app().await;
for (asked, sent) in [("br", Some("br")), ("zstd", Some("zstd")), ("gzip", Some("gzip")), ("identity", None)] {
// A page, not an empty feed list: under 32 bytes is sent as it is.
let req = axum::http::Request::builder()
.uri("/")
.header("accept-encoding", asked)
.body(Body::empty())
.unwrap();
let req =
axum::http::Request::builder().uri("/").header("accept-encoding", asked).body(Body::empty()).unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), 200);
let got = resp.headers().get("content-encoding").map(|v| v.to_str().unwrap().to_owned());
@@ -583,7 +616,11 @@ mod tests {
assert_eq!(call(&app, "GET", "/v1/feeds/nope", Some("ro"), None).await.0, 404);
assert_eq!(call(&app, "GET", "/v1/popular?limit=5", Some("ro"), None).await.1[0]["id"], id.as_str());
for (m, p) in [("DELETE", format!("/v1/feeds/{id}")), ("GET", "/v1/blocked".into()), ("DELETE", "/v1/instances/rw-token".into())] {
for (m, p) in [
("DELETE", format!("/v1/feeds/{id}")),
("GET", "/v1/blocked".into()),
("DELETE", "/v1/instances/rw-token".into()),
] {
assert_eq!(call(&app, m, &p, Some("rw"), None).await.0, 403, "{m} {p}");
}
assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), Some("rwa"), None).await.0, 204);

25
crates/directory/src/main.rs
View File

@@ -74,15 +74,24 @@ async fn main() -> Result<()> {
async fn serve(path: &std::path::Path) -> Result<()> {
// One JSON object a line for Loki when DIRECTORY_LOG_FORMAT=json, as ipx does; text otherwise.
let json = std::env::var("DIRECTORY_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
let filter = || tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
let filter =
|| tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
// Traces to Tempo when OTEL_EXPORTER_OTLP_ENDPOINT is set, joining ipx's (#213).
let otel = ipx_directory::otel::provider("ipx-directory")?;
tracing_subscriber::registry()
.with((!json).then(|| tracing_subscriber::fmt::layer().with_writer(std::io::stderr).with_filter(filter())))
.with(json.then(|| tracing_subscriber::fmt::layer().json().flatten_event(true).with_writer(std::io::stderr).with_filter(filter())))
.with(json.then(|| {
tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_writer(std::io::stderr)
.with_filter(filter())
}))
.with(otel.as_ref().map(|p| {
use opentelemetry::trace::TracerProvider;
tracing_opentelemetry::layer().with_tracer(p.tracer("ipx-directory")).with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info,ipx_feed=info"))
tracing_opentelemetry::layer()
.with_tracer(p.tracer("ipx-directory"))
.with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info,ipx_feed=info"))
}))
.init();
@@ -96,10 +105,13 @@ async fn serve(path: &std::path::Path) -> Result<()> {
let dir = Directory::new(store, vec![])?;
let dropped = dir.set_tokens(tokens).await?;
let listener = tokio::net::TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
let listener =
tokio::net::TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
tracing::info!(version = env!("CARGO_PKG_VERSION"), bind = %cfg.bind, tokens = cfg.tokens.len(), dropped, "directory started");
if !cfg.metrics_bind.is_empty() {
let metrics = tokio::net::TcpListener::bind(&cfg.metrics_bind).await.with_context(|| format!("listening for metrics on {}", cfg.metrics_bind))?;
let metrics = tokio::net::TcpListener::bind(&cfg.metrics_bind)
.await
.with_context(|| format!("listening for metrics on {}", cfg.metrics_bind))?;
tracing::info!(bind = %cfg.metrics_bind, "serving metrics for Prometheus at /metrics");
tokio::spawn(ipx_directory::metrics::serve(metrics));
}
@@ -112,7 +124,8 @@ async fn serve(path: &std::path::Path) -> Result<()> {
tokio::spawn(ipx_directory::reloader(dir.clone(), path.to_owned()));
axum::serve(listener, ipx_directory::router(dir))
.with_graceful_shutdown(async {
let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("SIGTERM handler");
let mut term =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("SIGTERM handler");
tokio::select! {
_ = term.recv() => {}
_ = tokio::signal::ctrl_c() => {}

57
crates/directory/src/metrics.rs
View File

@@ -14,7 +14,8 @@ static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
/// The process's resident memory in bytes and its open files, from /proc; None elsewhere.
pub fn process() -> (Option<f64>, Option<f64>) {
let rss = std::fs::read_to_string("/proc/self/status").ok().and_then(|s| {
let kb = s.lines().find_map(|l| l.strip_prefix("VmRSS:"))?.trim().strip_suffix("kB")?.trim().parse::<f64>().ok()?;
let kb =
s.lines().find_map(|l| l.strip_prefix("VmRSS:"))?.trim().strip_suffix("kB")?.trim().parse::<f64>().ok()?;
Some(kb * 1024.0)
});
let fds = std::fs::read_dir("/proc/self/fd").ok().map(|d| d.count() as f64);
@@ -35,10 +36,34 @@ fn render(started: std::time::Instant) -> String {
if let Some(v) = fds {
gauge("directory_process_open_fds", "gauge", "Files and sockets the process has open.", "", v);
}
gauge("directory_feed_fetches_total", "counter", "Feeds fetched, answered 304 included.", "", ipx_feed::FETCHES.load(Relaxed) as f64);
gauge("directory_feed_bytes_total", "counter", "Bytes of feed bodies fetched.", "", ipx_feed::FETCHED_BYTES.load(Relaxed) as f64);
gauge("directory_uptime_seconds", "gauge", "Seconds since the directory started.", "", started.elapsed().as_secs() as f64);
gauge("directory_build_info", "gauge", "The version running, as a label.", &format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")), 1.0);
gauge(
"directory_feed_fetches_total",
"counter",
"Feeds fetched, answered 304 included.",
"",
ipx_feed::FETCHES.load(Relaxed) as f64,
);
gauge(
"directory_feed_bytes_total",
"counter",
"Bytes of feed bodies fetched.",
"",
ipx_feed::FETCHED_BYTES.load(Relaxed) as f64,
);
gauge(
"directory_uptime_seconds",
"gauge",
"Seconds since the directory started.",
"",
started.elapsed().as_secs() as f64,
);
gauge(
"directory_build_info",
"gauge",
"The version running, as a label.",
&format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")),
1.0,
);
out
}
@@ -46,14 +71,15 @@ fn render(started: std::time::Instant) -> String {
/// Access in front, and these are for the LAN.
pub async fn serve(listener: tokio::net::TcpListener) {
let started = std::time::Instant::now();
let app = axum::Router::new().route(
"/metrics",
axum::routing::get(move || async move {
([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
}),
)
// Prometheus asks for gzip (#253).
.layer(tower_http::compression::CompressionLayer::new().gzip(true));
let app = axum::Router::new()
.route(
"/metrics",
axum::routing::get(move || async move {
([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
}),
)
// Prometheus asks for gzip (#253).
.layer(tower_http::compression::CompressionLayer::new().gzip(true));
if let Err(e) = axum::serve(listener, app).await {
tracing::error!(error = %e, "metrics stopped");
}
@@ -64,6 +90,9 @@ mod tests {
#[test]
fn metrics_say_memory_and_feeds() {
let text = super::render(std::time::Instant::now());
assert!(text.contains("directory_process_resident_bytes ") && text.contains("directory_feed_bytes_total "), "{text}");
assert!(
text.contains("directory_process_resident_bytes ") && text.contains("directory_feed_bytes_total "),
"{text}"
);
}
}

14
crates/directory/src/otel.rs
View File

@@ -70,13 +70,21 @@ mod tests {
assert_eq!(cx.span().span_context().trace_id().to_string(), "4bf92f3577b34da6a3ce929d0e0e4736");
assert!(cx.span().span_context().is_remote());
// Anything else from the browser starts a trace of its own.
for bad in ["", "00-xyz-00f067aa0ba902b7-01", "01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
"00-00000000000000000000000000000000-00f067aa0ba902b7-01", "00-4bf92f3577b34da6a3ce929d0e0e4736"] {
for bad in [
"",
"00-xyz-00f067aa0ba902b7-01",
"01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
"00-00000000000000000000000000000000-00f067aa0ba902b7-01",
"00-4bf92f3577b34da6a3ce929d0e0e4736",
] {
assert!(with(bad).is_none(), "{bad}");
}
assert!(remote_parent(&axum::http::HeaderMap::new(), None).is_none());
// From the query, where a beacon or Safari's player cannot send a header (#192).
let q = remote_parent(&axum::http::HeaderMap::new(), Some("x=1&traceparent=00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"));
let q = remote_parent(
&axum::http::HeaderMap::new(),
Some("x=1&traceparent=00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
);
assert_eq!(q.unwrap().span().span_context().trace_id().to_string(), "4bf92f3577b34da6a3ce929d0e0e4736");
assert!(remote_parent(&axum::http::HeaderMap::new(), Some("traceparent=nonsense")).is_none());
}

134
crates/directory/src/site.rs
View File

@@ -9,11 +9,11 @@
use std::collections::BTreeMap;
use std::fmt::Write;
use axum::Router;
use axum::extract::{Path, Query, State};
use axum::http::{HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::Router;
use crate::{Listing, Shared};
@@ -63,7 +63,9 @@ async fn home(State(dir): State<Shared>) -> Response {
Err(e) => return failed(e),
};
let mut top = all.clone();
top.sort_by(|a, b| b.subscribers.cmp(&a.subscribers).then_with(|| name(a).to_lowercase().cmp(&name(b).to_lowercase())));
top.sort_by(|a, b| {
b.subscribers.cmp(&a.subscribers).then_with(|| name(a).to_lowercase().cmp(&name(b).to_lowercase()))
});
top.truncate(10);
let mut cats: BTreeMap<&str, Vec<&Listing>> = BTreeMap::new();
for l in &all {
@@ -75,12 +77,25 @@ async fn home(State(dir): State<Shared>) -> Response {
} else {
b.push_str("<h2>Most subscribed</h2><ol class=chart>");
for (i, l) in top.iter().enumerate() {
let _ = write!(b, "<li><a href=\"/feed/{}\"><span class=no>{}</span>{}<span class=txt><b>{}</b><small>{}</small></span></a></li>",
esc(&l.id), i + 1, art(l, "art sm"), esc(name(l)), subscribers(l.subscribers));
let _ = write!(
b,
"<li><a href=\"/feed/{}\"><span class=no>{}</span>{}<span class=txt><b>{}</b><small>{}</small></span></a></li>",
esc(&l.id),
i + 1,
art(l, "art sm"),
esc(name(l)),
subscribers(l.subscribers)
);
}
b.push_str("</ol><h2>Categories</h2><div class=cats>");
for (c, ls) in &cats {
let _ = write!(b, "<a class=cat href=\"/category/{}\"><b>{}</b><small>{}</small></a>", seg(c), esc(c), ls.len());
let _ = write!(
b,
"<a class=cat href=\"/category/{}\"><b>{}</b><small>{}</small></a>",
seg(c),
esc(c),
ls.len()
);
}
let _ = write!(b, "</div><p><a class=more href=\"/all\">All {} feeds, A to Z</a></p>", all.len());
}
@@ -115,10 +130,11 @@ async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn
if ls.is_empty() && here.starts_with("/category/") {
return not_found("That category has nothing in it.");
}
let subs: BTreeMap<String, usize> = ls.iter().filter_map(|l| l.subcategory.clone()).fold(BTreeMap::new(), |mut m, s| {
*m.entry(s).or_default() += 1;
m
});
let subs: BTreeMap<String, usize> =
ls.iter().filter_map(|l| l.subcategory.clone()).fold(BTreeMap::new(), |mut m, s| {
*m.entry(s).or_default() += 1;
m
});
ls.retain(|l| f.keep(l) && f.sub.as_ref().is_none_or(|s| l.subcategory.as_ref() == Some(s)));
ls.sort_by_key(|l| name(l).to_lowercase());
@@ -138,18 +154,38 @@ async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn
if q.is_empty() { here.to_owned() } else { format!("{here}?{q}") }
};
let mut b = String::new();
let _ = write!(b, "<p class=back><a href=\"/\">Directory</a></p><h2>{}</h2><nav class=tabs aria-label=Kind>", esc(title));
let _ = write!(
b,
"<p class=back><a href=\"/\">Directory</a></p><h2>{}</h2><nav class=tabs aria-label=Kind>",
esc(title)
);
for (k, label) in [(None, "All"), (Some("podcasts"), "Podcasts"), (Some("blogs"), "Blogs")] {
let on = f.kind.as_deref() == k;
let _ = write!(b, "<a href=\"{}\"{}>{label}</a>", esc(&link(k, f.sub.as_deref())), if on { " aria-current=page" } else { "" });
let _ = write!(
b,
"<a href=\"{}\"{}>{label}</a>",
esc(&link(k, f.sub.as_deref())),
if on { " aria-current=page" } else { "" }
);
}
b.push_str("</nav>");
if !subs.is_empty() {
b.push_str("<nav class=pills aria-label=Subcategory>");
let _ = write!(b, "<a href=\"{}\"{}>All</a>", esc(&link(f.kind.as_deref(), None)), if f.sub.is_none() { " aria-current=page" } else { "" });
let _ = write!(
b,
"<a href=\"{}\"{}>All</a>",
esc(&link(f.kind.as_deref(), None)),
if f.sub.is_none() { " aria-current=page" } else { "" }
);
for (s, n) in &subs {
let on = f.sub.as_deref() == Some(s.as_str());
let _ = write!(b, "<a href=\"{}\"{}>{} <small>{n}</small></a>", esc(&link(f.kind.as_deref(), Some(s))), if on { " aria-current=page" } else { "" }, esc(s));
let _ = write!(
b,
"<a href=\"{}\"{}>{} <small>{n}</small></a>",
esc(&link(f.kind.as_deref(), Some(s))),
if on { " aria-current=page" } else { "" },
esc(s)
);
}
b.push_str("</nav>");
}
@@ -158,7 +194,14 @@ async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn
} else {
let _ = write!(b, "<p class=count>{}</p><div class=tiles>", plural(ls.len(), "feed"));
for l in &ls {
let _ = write!(b, "<a class=tile href=\"/feed/{}\">{}<b>{}</b><small>{}</small></a>", esc(&l.id), art(l, "art"), esc(name(l)), subscribers(l.subscribers));
let _ = write!(
b,
"<a class=tile href=\"/feed/{}\">{}<b>{}</b><small>{}</small></a>",
esc(&l.id),
art(l, "art"),
esc(name(l)),
subscribers(l.subscribers)
);
}
b.push_str("</div>");
}
@@ -177,11 +220,21 @@ async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Response {
if let Some(c) = &l.category {
let _ = write!(b, " <span aria-hidden=true>/</span> <a href=\"/category/{}\">{}</a>", seg(c), esc(c));
}
let _ = write!(b, "</p><div class=show>{}<div class=meta><h2>{}</h2><p class=sub>{}{}</p>", art(l, "art lg"), esc(name(l)),
subscribers(l.subscribers), l.subcategory.as_deref().map(|s| format!(" &middot; {}", esc(s))).unwrap_or_default());
let _ = write!(b, "<div class=addr><label for=addr>Feed address</label><span class=row><input id=addr readonly value=\"{}\" spellcheck=false>\
let _ = write!(
b,
"</p><div class=show>{}<div class=meta><h2>{}</h2><p class=sub>{}{}</p>",
art(l, "art lg"),
esc(name(l)),
subscribers(l.subscribers),
l.subcategory.as_deref().map(|s| format!(" &middot; {}", esc(s))).unwrap_or_default()
);
let _ = write!(
b,
"<div class=addr><label for=addr>Feed address</label><span class=row><input id=addr readonly value=\"{}\" spellcheck=false>\
<button type=button data-copy=\"{}\">Copy</button></span><small>Paste it into any podcast app or feed reader to subscribe.</small></div></div></div>",
esc(&l.url), esc(&l.url));
esc(&l.url),
esc(&l.url)
);
if let Some(d) = p.description.as_deref().map(text).filter(|d| !d.is_empty() && d != name(l)) {
let _ = write!(b, "<p class=about>{}</p>", esc(&d));
}
@@ -199,17 +252,26 @@ async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Response {
None => esc(&title),
};
let blurb: String = e.description.as_deref().map(text).unwrap_or_default().chars().take(400).collect();
let when = [e.published.map(date), e.duration.filter(|d| *d > 0).map(clock)].into_iter().flatten().collect::<Vec<_>>().join(" &middot; ");
let _ = write!(b, "<article><b>{head}</b>{}{}</article>",
let when = [e.published.map(date), e.duration.filter(|d| *d > 0).map(clock)]
.into_iter()
.flatten()
.collect::<Vec<_>>()
.join(" &middot; ");
let _ = write!(
b,
"<article><b>{head}</b>{}{}</article>",
if blurb.is_empty() { String::new() } else { format!("<p>{}</p>", esc(&blurb)) },
if when.is_empty() { String::new() } else { format!("<small>{when}</small>") });
if when.is_empty() { String::new() } else { format!("<small>{when}</small>") }
);
}
page(name(l), Some(&text(p.description.as_deref().unwrap_or(""))), &b)
}
fn page(title: &str, description: Option<&str>, body: &str) -> Response {
let full = if title == "iPX Directory" { title.to_owned() } else { format!("{title} | iPX Directory") };
let desc = description.filter(|d| !d.is_empty()).map(|d| d.chars().take(200).collect::<String>())
let desc = description
.filter(|d| !d.is_empty())
.map(|d| d.chars().take(200).collect::<String>())
.unwrap_or_else(|| "Public podcasts and blogs that iPX servers share.".into());
let html = format!(
"<!doctype html><html lang=en><head><meta charset=utf-8><meta name=viewport content=\"width=device-width,initial-scale=1\">\
@@ -234,14 +296,22 @@ fn page(title: &str, description: Option<&str>, body: &str) -> Response {
}
fn not_found(why: &str) -> Response {
let mut r = page("Not found", None, &format!("<p class=back><a href=\"/\">Directory</a></p><h2>Not found</h2><p class=hint>{}</p>", esc(why)));
let mut r = page(
"Not found",
None,
&format!("<p class=back><a href=\"/\">Directory</a></p><h2>Not found</h2><p class=hint>{}</p>", esc(why)),
);
*r.status_mut() = StatusCode::NOT_FOUND;
r
}
fn failed(e: anyhow::Error) -> Response {
tracing::error!(error = %format!("{e:#}"), "a directory page failed");
let mut r = page("Unavailable", None, "<h2>Unavailable</h2><p class=hint>The directory could not read its list just now. Try again in a minute.</p>");
let mut r = page(
"Unavailable",
None,
"<h2>Unavailable</h2><p class=hint>The directory could not read its list just now. Try again in a minute.</p>",
);
*r.status_mut() = StatusCode::SERVICE_UNAVAILABLE;
r
}
@@ -253,8 +323,12 @@ fn redirect(to: &'static str) -> Response {
/// A cover from its publisher, over the feed's initials, which show where it has none or it fails.
fn art(l: &Listing, class: &str) -> String {
// The first letter of each of the first two words that have one: `<antirez>` is A, not nothing.
let initials: String = name(l).split_whitespace().filter_map(|w| w.chars().find(|c| c.is_alphanumeric())).take(2).collect();
let img = l.image.as_deref().filter(|u| u.starts_with("https://") || u.starts_with("http://"))
let initials: String =
name(l).split_whitespace().filter_map(|w| w.chars().find(|c| c.is_alphanumeric())).take(2).collect();
let img = l
.image
.as_deref()
.filter(|u| u.starts_with("https://") || u.starts_with("http://"))
.map(|u| format!("<img src=\"{}\" alt=\"\" loading=lazy referrerpolicy=no-referrer>", esc(u)))
.unwrap_or_default();
format!("<span class=\"{class}\" aria-hidden=true><i>{}</i>{img}</span>", esc(&initials.to_uppercase()))
@@ -301,7 +375,13 @@ fn clock(secs: i64) -> String {
fn text(html: &str) -> String {
// A space where each tag was, or two paragraphs run together: "drives.For over".
let stripped = ammonia::Builder::empty().clean(&html.replace('<', " <")).to_string();
let decoded = stripped.replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", "\"").replace("&#39;", "'").replace("&nbsp;", " ").replace("&amp;", "&");
let decoded = stripped
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&nbsp;", " ")
.replace("&amp;", "&");
decoded.split_whitespace().collect::<Vec<_>>().join(" ")
}

72
crates/directory/src/store.rs
View File

@@ -28,11 +28,7 @@ pub struct Due {
/// What a fetch found, to be stored.
pub enum Refreshed {
NotModified,
Feed {
parsed: ipx_feed::ParsedFeed,
etag: Option<String>,
last_modified: Option<String>,
},
Feed { parsed: ipx_feed::ParsedFeed, etag: Option<String>, last_modified: Option<String> },
Failed(String),
}
@@ -122,7 +118,12 @@ impl Store {
let schema = format!("ipxd_{pid}_{n}");
admin.db.execute_unprepared(&format!("CREATE SCHEMA {schema}")).await.unwrap();
let sep = if url.contains('?') { '&' } else { '?' };
Self::open(&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning"), false).await.unwrap()
Self::open(
&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning"),
false,
)
.await
.unwrap()
}
/// `keep_open` for the server, which keeps its whole pool connected.
@@ -195,8 +196,12 @@ impl Store {
/// blocked feed is left out and counted; the rest are added to the feeds, new ones to be
/// fetched by the refresher. Feeds no instance sends any more go.
pub async fn snapshot(&self, instance: &str, entries: &[SnapshotEntry]) -> Result<SnapshotResult> {
let blocked: std::collections::HashSet<String> =
self.rows("SELECT id FROM blocked", vec![]).await?.iter().map(|r| r.try_get("", "id")).collect::<Result<_, _>>()?;
let blocked: std::collections::HashSet<String> = self
.rows("SELECT id FROM blocked", vec![])
.await?
.iter()
.map(|r| r.try_get("", "id"))
.collect::<Result<_, _>>()?;
let mut result = SnapshotResult::default();
let mut keep = std::collections::BTreeMap::new();
for e in entries {
@@ -372,7 +377,13 @@ impl Store {
)
.await?
.iter()
.map(|r| Ok(Instance { name: r.try_get("", "instance")?, feeds: r.try_get("", "feeds")?, subscribers: r.try_get("", "subscribers")? }))
.map(|r| {
Ok(Instance {
name: r.try_get("", "instance")?,
feeds: r.try_get("", "feeds")?,
subscribers: r.try_get("", "subscribers")?,
})
})
.collect()
}
@@ -380,7 +391,14 @@ impl Store {
self.rows("SELECT id, url, title, at FROM blocked ORDER BY at DESC, id", vec![])
.await?
.iter()
.map(|r| Ok(Blocked { id: r.try_get("", "id")?, url: r.try_get("", "url")?, title: r.try_get("", "title")?, at: r.try_get("", "at")? }))
.map(|r| {
Ok(Blocked {
id: r.try_get("", "id")?,
url: r.try_get("", "url")?,
title: r.try_get("", "title")?,
at: r.try_get("", "at")?,
})
})
.collect()
}
@@ -425,7 +443,11 @@ impl Store {
let t = now();
match got {
Refreshed::NotModified => {
self.exec("UPDATE feeds SET checked_at = $1, last_error = NULL, error_since = NULL WHERE id = $2", vec![t.into(), id.into()]).await?;
self.exec(
"UPDATE feeds SET checked_at = $1, last_error = NULL, error_since = NULL WHERE id = $2",
vec![t.into(), id.into()],
)
.await?;
}
Refreshed::Failed(err) => {
self.exec(
@@ -436,16 +458,25 @@ impl Store {
}
Refreshed::Feed { parsed, etag, last_modified } => {
let path = parsed.category.as_deref().map(ipx_feed::category_path);
let podcast = parsed.entries.iter().flat_map(|e| &e.enclosures).any(|e| {
e.mime.as_deref().is_some_and(|m| m.starts_with("audio/") || m.starts_with("video/"))
});
let podcast = parsed
.entries
.iter()
.flat_map(|e| &e.enclosures)
.any(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("audio/") || m.starts_with("video/")));
let mut entries = parsed.entries;
entries.sort_by_key(|e| std::cmp::Reverse(e.published));
// What an item is, never its files: their addresses are where a key would be.
let items: Vec<Item> = entries
.into_iter()
.take(ITEMS)
.map(|e| Item { guid: e.guid, title: e.title, link: e.link, published: e.published, duration: e.duration, description: e.description })
.map(|e| Item {
guid: e.guid,
title: e.title,
link: e.link,
published: e.published,
duration: e.duration,
description: e.description,
})
.collect();
self.exec(
"UPDATE feeds SET title = $1, image = $2, category = $3, subcategory = $4, description = $5,
@@ -541,7 +572,10 @@ pub(crate) mod tests {
assert_eq!(all[0].subscribers, 3);
let i = s.instances().await.unwrap();
assert_eq!(i.iter().map(|i| (i.name.as_str(), i.feeds, i.subscribers)).collect::<Vec<_>>(), [("a", 1, 1), ("b", 1, 2)]);
assert_eq!(
i.iter().map(|i| (i.name.as_str(), i.feeds, i.subscribers)).collect::<Vec<_>>(),
[("a", 1, 1), ("b", 1, 2)]
);
// A token taken out of the config file takes its counts.
s.keep_instances(&["b".into()]).await.unwrap();
@@ -583,7 +617,11 @@ pub(crate) mod tests {
#[tokio::test]
async fn a_feed_naming_no_category_takes_the_one_a_server_gave_it() {
let s = Store::scratch().await;
let entry = |c: Option<&str>| SnapshotEntry { url: "https://x.test/blog.xml".into(), subscribers: 1, category: c.map(Into::into) };
let entry = |c: Option<&str>| SnapshotEntry {
url: "https://x.test/blog.xml".into(),
subscribers: 1,
category: c.map(Into::into),
};
s.snapshot("a", &[entry(Some("Visual Arts"))]).await.unwrap();
let d = s.due(0, 10).await.unwrap().remove(0);
let mut parsed = ipx_feed::parse(include_bytes!("../../../tests/data/rss2.xml")).unwrap();

262
crates/feed/src/lib.rs
View File

@@ -158,9 +158,8 @@ pub async fn fetch(
// The original surfaced 401/407 specially; the code is enough for a UI to switch on.
return Err(anyhow!("HTTP {status}"));
}
let header = |h: reqwest::header::HeaderName| {
resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned)
};
let header =
|h: reqwest::header::HeaderName| resp.headers().get(&h).and_then(|v| v.to_str().ok()).map(str::to_owned);
let etag = header(ETAG);
let last_modified = header(LAST_MODIFIED);
let bytes = resp.bytes().await.context("reading body")?.to_vec();
@@ -188,7 +187,10 @@ pub fn failure_kind(msg: &str) -> (String, Option<u16>) {
if let Some(c) = code.filter(|c| (100..600).contains(c)) {
return (c.to_string(), Some(c));
}
let kind = if low.contains("dns error") || low.contains("failed to lookup address") || low.contains("no address associated") {
let kind = if low.contains("dns error")
|| low.contains("failed to lookup address")
|| low.contains("no address associated")
{
"dns"
} else if low.contains("too many redirects") {
"redirect_loop"
@@ -222,14 +224,14 @@ pub fn failure_kind(msg: &str) -> (String, Option<u16>) {
/// nothing extra, so a miss costs a clearer message, not a wrong one.
pub fn explain_failure(msg: &str) -> Option<Failure> {
if let Some(rest) = msg.strip_prefix("got a web page, not a feed") {
let new_url = rest
.strip_prefix("; it links ")
.and_then(|r| r.strip_suffix(" as its feed"))
.map(str::to_owned);
let new_url = rest.strip_prefix("; it links ").and_then(|r| r.strip_suffix(" as its feed")).map(str::to_owned);
return Some(Failure { reason: "The feed moved; this address now shows a web page.", new_url });
}
if msg.contains("the site sent ") {
return Some(Failure { reason: "The site sent a message instead of the feed; the publisher has to fix it.", new_url: None });
return Some(Failure {
reason: "The site sent a message instead of the feed; the publisher has to fix it.",
new_url: None,
});
}
let low = msg.to_ascii_lowercase();
if low.contains("http 404") {
@@ -247,7 +249,9 @@ pub fn explain_failure(msg: &str) -> Option<Failure> {
// Only a name that does not exist. A lookup that failed for the moment says nothing about the
// site, and NPR's and Megaphone's feeds were labelled gone after a DNS outage (#143).
if !is_temporary_lookup_failure(msg)
&& (low.contains("dns error") || low.contains("failed to lookup address") || low.contains("no address associated"))
&& (low.contains("dns error")
|| low.contains("failed to lookup address")
|| low.contains("no address associated"))
{
return Some(Failure { reason: "This address no longer resolves; the site may be gone.", new_url: None });
}
@@ -276,8 +280,7 @@ pub fn is_opml(bytes: &[u8]) -> bool {
/// The feeds listed in an OPML document, as (title, xml_url), walking nested folders.
pub fn parse_opml(bytes: &[u8]) -> Result<Vec<(String, String)>> {
let text = String::from_utf8_lossy(bytes);
let doc = opml::OPML::from_str(&text)
.map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
let doc = opml::OPML::from_str(&text).map_err(|e| anyhow!("that does not parse as OPML: {e}"))?;
let mut out = vec![];
collect_outlines(&doc.body.outlines, &mut out);
Ok(out)
@@ -298,10 +301,7 @@ pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, Strin
pub fn opml_title(bytes: &[u8]) -> Option<String> {
let text = String::from_utf8_lossy(bytes);
let doc = opml::OPML::from_str(&text).ok()?;
doc.head
.and_then(|h| h.title)
.map(|t| t.trim().to_owned())
.filter(|t| !t.is_empty())
doc.head.and_then(|h| h.title).map(|t| t.trim().to_owned()).filter(|t| !t.is_empty())
}
/// The token and show of a Patreon feed link, or None for any other URL.
@@ -351,10 +351,7 @@ pub fn same_feed(a: &str, b: &str) -> bool {
/// ponytail: Patreon's own web API, undocumented, asked without signing in. If it changes,
/// finding shows stops and the show feeds already found keep working. The documented API
/// needs an OAuth client per install and does not list shows.
pub async fn patreon_shows(
client: &reqwest::Client,
url: &str,
) -> Result<(Option<String>, Vec<(String, String)>)> {
pub async fn patreon_shows(client: &reqwest::Client, url: &str) -> Result<(Option<String>, Vec<(String, String)>)> {
// The creator feed names its campaign by number in its self link, a few hundred bytes in.
// The whole feed runs to megabytes and Patreon ignores Range, so read until it turns up.
let mut resp = client.get(url).timeout(FEED_TIMEOUT).send().await.context("connecting")?;
@@ -366,8 +363,7 @@ pub async fn patreon_shows(
let Some(chunk) = resp.chunk().await.context("reading the feed")? else { break };
head.extend_from_slice(&chunk);
}
let campaign = patreon_campaign(&head)
.ok_or_else(|| anyhow!("the Patreon feed does not say whose it is"))?;
let campaign = patreon_campaign(&head).ok_or_else(|| anyhow!("the Patreon feed does not say whose it is"))?;
let api = format!(
"https://www.patreon.com/api/campaigns/{campaign}\
@@ -490,7 +486,8 @@ fn alternate_feed_link(bytes: &[u8]) -> Option<String> {
let tag_lower = &lower[start..end];
let is_alternate = tag_lower.contains("rel=\"alternate\"") || tag_lower.contains("rel='alternate'");
let is_feed_type = tag_lower.contains("rss+xml") || tag_lower.contains("atom+xml");
if is_alternate && is_feed_type
if is_alternate
&& is_feed_type
&& let Some(href) = tag_attr(tag, "href")
{
return Some(href);
@@ -505,12 +502,7 @@ fn alternate_feed_link(bytes: &[u8]) -> Option<String> {
/// feed that moved (#102); cnn.com is one.
pub async fn find_feed(client: &reqwest::Client, url: &str) -> Result<String> {
let read = |u: String| async move {
let resp = client
.get(&u)
.timeout(std::time::Duration::from_secs(30))
.send()
.await
.context("connecting")?;
let resp = client.get(&u).timeout(std::time::Duration::from_secs(30)).send().await.context("connecting")?;
anyhow::ensure!(resp.status().is_success(), "HTTP {}", resp.status());
let base = resp.url().clone();
anyhow::Ok((base, resp.bytes().await.context("reading")?))
@@ -527,7 +519,8 @@ pub async fn find_feed(client: &reqwest::Client, url: &str) -> Result<String> {
let Some(href) = alternate_feed_link(&body) else {
anyhow::bail!("{url} is a web page that links no feed, so there is nothing to subscribe to");
};
let linked = base.join(&href).with_context(|| format!("{url} links {href} as its feed, which is not an address"))?;
let linked =
base.join(&href).with_context(|| format!("{url} links {href} as its feed, which is not an address"))?;
let (_, feed) = read(linked.to_string()).await.with_context(|| format!("{url} links {linked} as its feed, but"))?;
anyhow::ensure!(is_feed(&feed), "{url} links {linked} as its feed, but that is not a feed either");
Ok(linked.into())
@@ -682,9 +675,7 @@ fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> {
let (mut url, mut mime, mut length) = (String::new(), None, None);
for attr in e.attributes().flatten() {
// Values arrive escaped: a feed URL's "&" is "&amp;" in the document.
let val = quick_xml::escape::unescape(&attr.value)
.map(|v| v.trim().to_string())
.unwrap_or_default();
let val = quick_xml::escape::unescape(&attr.value).map(|v| v.trim().to_string()).unwrap_or_default();
match attr.key.local_name().as_ref() {
"url" => url = val,
"type" => mime = Some(val).filter(|v| !v.is_empty()),
@@ -736,10 +727,7 @@ fn enclosures_by_item(bytes: &[u8]) -> Vec<Vec<Enclosure>> {
fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
let per_item = enclosures_by_item(bytes);
let explicit = ch
.itunes_ext()
.and_then(|it| it.explicit())
.is_some_and(is_yes);
let explicit = ch.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
let entries = ch
.items()
@@ -768,10 +756,7 @@ fn from_rss(ch: rss::Channel, bytes: &[u8]) -> ParsedFeed {
item.title(),
)?;
let entry_explicit = item
.itunes_ext()
.and_then(|it| it.explicit())
.is_some_and(is_yes);
let entry_explicit = item.itunes_ext().and_then(|it| it.explicit()).is_some_and(is_yes);
let it = item.itunes_ext();
Some(Entry {
@@ -839,18 +824,10 @@ fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
.collect();
drop_player_repeats(&mut enclosures);
let alt = e
.links()
.iter()
.find(|l| l.rel() == "alternate" || l.rel().is_empty())
.map(|l| l.href());
let alt = e.links().iter().find(|l| l.rel() == "alternate" || l.rel().is_empty()).map(|l| l.href());
let guid = pick_guid(
Some(e.id()),
alt,
enclosures.first().map(|x| x.url.as_str()),
Some(e.title().as_str()),
)?;
let guid =
pick_guid(Some(e.id()), alt, enclosures.first().map(|x| x.url.as_str()), Some(e.title().as_str()))?;
Some(Entry {
guid,
@@ -888,18 +865,8 @@ fn from_atom(feed: atom_syndication::Feed) -> ParsedFeed {
/// The original fell back to hashing the title or description. A guid, permalink or
/// enclosure URL is a stable identifier already, so no hashing is needed; an entry with
/// none of them has nothing to download and is dropped.
fn pick_guid(
guid: Option<&str>,
link: Option<&str>,
enclosure: Option<&str>,
title: Option<&str>,
) -> Option<String> {
[guid, link, enclosure, title]
.into_iter()
.flatten()
.map(str::trim)
.find(|s| !s.is_empty())
.map(str::to_owned)
fn pick_guid(guid: Option<&str>, link: Option<&str>, enclosure: Option<&str>, title: Option<&str>) -> Option<String> {
[guid, link, enclosure, title].into_iter().flatten().map(str::trim).find(|s| !s.is_empty()).map(str::to_owned)
}
fn is_yes(s: &str) -> bool {
@@ -946,9 +913,7 @@ fn title_text(s: Option<&str>) -> Option<String> {
while let Some(at) = rest.find('&') {
out.push_str(&rest[..at]);
rest = &rest[at..];
let len = 1 + rest[1..]
.find(|c: char| !(c.is_ascii_alphanumeric() || c == '#'))
.unwrap_or(rest.len() - 1);
let len = 1 + rest[1..].find(|c: char| !(c.is_ascii_alphanumeric() || c == '#')).unwrap_or(rest.len() - 1);
let decoded = rest[len..]
.starts_with(';')
.then(|| quick_xml::escape::unescape_with(&rest[..=len], quick_xml::escape::resolve_html5_entity).ok())
@@ -1014,7 +979,10 @@ fn text_of(html: &str) -> String {
for c in html.chars() {
match c {
'<' => in_tag = true,
'>' if in_tag => { in_tag = false; out.push(' '); }
'>' if in_tag => {
in_tag = false;
out.push(' ');
}
_ if !in_tag => out.push(c),
_ => {}
}
@@ -1051,14 +1019,9 @@ fn item_image(item: &rss::Item, enclosures: &[Enclosure]) -> Option<String> {
non_empty(e.attrs.get("url").map(String::as_str))
})
};
attr("thumbnail", false)
.or_else(|| attr("content", true))
.or_else(|| {
enclosures
.iter()
.find(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("image/")))
.map(|e| e.url.clone())
})
attr("thumbnail", false).or_else(|| attr("content", true)).or_else(|| {
enclosures.iter().find(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("image/"))).map(|e| e.url.clone())
})
}
/// itunes:duration is either plain seconds ("5649") or a clock ("1:34:09", "23:45").
@@ -1099,13 +1062,62 @@ const CATEGORIES: &[(&str, &[&str])] = &[
("History", &[]),
("Health & Fitness", &["Alternative Health", "Fitness", "Medicine", "Mental Health", "Nutrition", "Sexuality"]),
("Kids & Family", &["Education for Kids", "Parenting", "Pets & Animals", "Stories for Kids"]),
("Leisure", &["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"]),
(
"Leisure",
&["Animation & Manga", "Automotive", "Aviation", "Crafts", "Games", "Hobbies", "Home & Garden", "Video Games"],
),
("Music", &["Music Commentary", "Music History", "Music Interviews"]),
("News", &["Business News", "Daily News", "Entertainment News", "News Commentary", "Politics", "Sports News", "Tech News"]),
("Religion & Spirituality", &["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"]),
("Science", &["Astronomy", "Chemistry", "Earth Sciences", "Life Sciences", "Mathematics", "Natural Sciences", "Nature", "Physics", "Social Sciences"]),
(
"News",
&[
"Business News",
"Daily News",
"Entertainment News",
"News Commentary",
"Politics",
"Sports News",
"Tech News",
],
),
(
"Religion & Spirituality",
&["Buddhism", "Christianity", "Hinduism", "Islam", "Judaism", "Religion", "Spirituality"],
),
(
"Science",
&[
"Astronomy",
"Chemistry",
"Earth Sciences",
"Life Sciences",
"Mathematics",
"Natural Sciences",
"Nature",
"Physics",
"Social Sciences",
],
),
("Society & Culture", &["Documentary", "Personal Journals", "Philosophy", "Places & Travel", "Relationships"]),
("Sports", &["Baseball", "Basketball", "Cricket", "Fantasy Sports", "Football", "Golf", "Hockey", "Rugby", "Running", "Soccer", "Swimming", "Tennis", "Volleyball", "Wilderness", "Wrestling"]),
(
"Sports",
&[
"Baseball",
"Basketball",
"Cricket",
"Fantasy Sports",
"Football",
"Golf",
"Hockey",
"Rugby",
"Running",
"Soccer",
"Swimming",
"Tennis",
"Volleyball",
"Wilderness",
"Wrestling",
],
),
("Technology", &[]),
("True Crime", &[]),
("TV & Film", &["After Shows", "Film History", "Film Interviews", "Film Reviews", "TV Reviews"]),
@@ -1207,15 +1219,12 @@ pub fn url_looks_private(feed_url: &str) -> bool {
|| u.password().is_some()
|| u.query_pairs().any(|(k, _)| {
let k = k.to_ascii_lowercase();
["auth", "token", "key", "secret", "pass", "sig", "session", "user", "uid"]
.iter()
.any(|w| k.contains(w))
["auth", "token", "key", "secret", "pass", "sig", "session", "user", "uid"].iter().any(|w| k.contains(w))
})
}
/// Services whose feeds are always one subscriber's own.
const PAID_HOSTS: &[&str] =
&["patreon.com", "supercast.com", "supportingcast.fm", "glow.fm", "memberful.com"];
const PAID_HOSTS: &[&str] = &["patreon.com", "supercast.com", "supportingcast.fm", "glow.fm", "memberful.com"];
#[cfg(test)]
mod tests {
@@ -1227,7 +1236,9 @@ mod tests {
assert_eq!(super::redact("not a url?token=SECRET"), "not a url");
// As reqwest words a failure, and with two addresses and none.
assert_eq!(
super::redact_text("connecting: error sending request for url (https://www.patreon.com/rss/x?auth=SECRET&show=1): operation timed out"),
super::redact_text(
"connecting: error sending request for url (https://www.patreon.com/rss/x?auth=SECRET&show=1): operation timed out"
),
"connecting: error sending request for url (https://www.patreon.com/rss/x): operation timed out"
);
assert_eq!(super::redact_text("from http://a/f?k=1 to https://u:p@b/g"), "from http://a/f to https://b/g");
@@ -1266,10 +1277,16 @@ mod tests {
let path = req.split_whitespace().nth(1).unwrap_or("/").to_owned();
let body = "<?xml version=\"1.0\"?><rss version=\"2.0\"><channel><title>T</title></channel></rss>";
let resp = match path.as_str() {
"/old" => "HTTP/1.1 301 Moved Permanently\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(),
"/old" => {
"HTTP/1.1 301 Moved Permanently\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned()
}
"/tmp" => "HTTP/1.1 302 Found\r\nLocation: /new\r\nContent-Length: 0\r\n\r\n".to_owned(),
"/chain" => "HTTP/1.1 308 Permanent Redirect\r\nLocation: /tmp\r\nContent-Length: 0\r\n\r\n".to_owned(),
"/loop" => "HTTP/1.1 301 Moved Permanently\r\nLocation: /loop\r\nContent-Length: 0\r\n\r\n".to_owned(),
"/chain" => {
"HTTP/1.1 308 Permanent Redirect\r\nLocation: /tmp\r\nContent-Length: 0\r\n\r\n".to_owned()
}
"/loop" => {
"HTTP/1.1 301 Moved Permanently\r\nLocation: /loop\r\nContent-Length: 0\r\n\r\n".to_owned()
}
_ => format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()),
};
let _ = sock.write_all(resp.as_bytes()).await;
@@ -1291,9 +1308,9 @@ mod tests {
let (got, moved) = get("/old").await.unwrap();
assert!(matches!(got, super::Fetched::Body { .. }));
assert_eq!(moved, Some(format!("{base}/new")));
assert_eq!(get("/tmp").await.unwrap().1, None); // 302: for now
assert_eq!(get("/chain").await.unwrap().1, None); // 308 then 302: not for good
assert_eq!(get("/new").await.unwrap().1, None); // never moved
assert_eq!(get("/tmp").await.unwrap().1, None); // 302: for now
assert_eq!(get("/chain").await.unwrap().1, None); // 308 then 302: not for good
assert_eq!(get("/new").await.unwrap().1, None); // never moved
let looped = get("/loop").await.err().unwrap().to_string();
assert_eq!(super::failure_kind(&looped).0, "redirect_loop");
}
@@ -1327,10 +1344,7 @@ mod tests {
);
assert!(feed.entries[1].explicit, "entry-level itunes:explicit=yes");
assert_eq!(
feed.entries[2].enclosures[0].mime.as_deref(),
Some("application/x-bittorrent")
);
assert_eq!(feed.entries[2].enclosures[0].mime.as_deref(), Some("application/x-bittorrent"));
}
#[test]
@@ -1352,9 +1366,12 @@ mod tests {
<enclosure url="https://x/ep.mp3?_=2" length="1" type="audio/mpeg"/>
<enclosure url="https://x/other.mp3?_=3&amp;key=k" length="1" type="audio/mpeg"/>
</item></channel></rss>"#;
let urls: Vec<String> =
parse(xml).unwrap().entries[0].enclosures.iter().map(|e| e.url.clone()).collect();
assert_eq!(urls, ["https://x/ep.mp3", "https://x/other.mp3?_=3&key=k"], "the repeat goes, a different file stays");
let urls: Vec<String> = parse(xml).unwrap().entries[0].enclosures.iter().map(|e| e.url.clone()).collect();
assert_eq!(
urls,
["https://x/ep.mp3", "https://x/other.mp3?_=3&key=k"],
"the repeat goes, a different file stays"
);
assert_eq!(same_file_key("https://x/a.mp3?key=k&_=2"), same_file_key("https://x/a.mp3?key=k"));
assert_ne!(same_file_key("https://x/a.mp3?_=x"), same_file_key("https://x/a.mp3"), "only a number");
}
@@ -1381,8 +1398,15 @@ mod tests {
let cut = r#"*]:pointer-events-auto R6Vx5W_threadScrollVars" dir="auto" data-turn="assistant"> <p>What if</p>"#;
let whole = r#"<div class="[&:has([data-writing-block])>*]:pointer-events-auto"><p>What if</p></div>"#;
assert_eq!(body(Some(cut), Some(whole)).as_deref(), Some(whole));
assert_eq!(body(Some("<p>Notes</p>"), Some("<p>Notes</p>")).as_deref(), Some("<p>Notes</p>"), "a whole body wins");
assert_eq!(body(Some("Plain notes, no tags."), Some("Plain notes, no tags.")).as_deref(), Some("Plain notes, no tags."));
assert_eq!(
body(Some("<p>Notes</p>"), Some("<p>Notes</p>")).as_deref(),
Some("<p>Notes</p>"),
"a whole body wins"
);
assert_eq!(
body(Some("Plain notes, no tags."), Some("Plain notes, no tags.")).as_deref(),
Some("Plain notes, no tags.")
);
assert_eq!(body(Some(cut), None).as_deref(), Some(cut), "a damaged body beats none");
assert_eq!(body(None, Some("Summary")).as_deref(), Some("Summary"));
}
@@ -1401,14 +1425,19 @@ mod tests {
);
// A podcast repeating its notes, whole, cut short, or differently spaced: shown once.
let notes = "<p>This week we talk about <a href=\"x\">tape</a>, drums and a very long list.</p>";
for d in ["This week we talk about tape, drums and a very long list.",
"This week we talk about tape, drums…",
"This week we talk about\ntape [...]"] {
for d in [
"This week we talk about tape, drums and a very long list.",
"This week we talk about tape, drums…",
"This week we talk about\ntape [...]",
] {
assert_eq!(body(Some(notes), Some(d)).as_deref(), Some(notes), "{d:?} is already in the body");
}
let long = "word ".repeat(80);
assert_eq!(body(Some("<p>The post.</p>"), Some(&long)).as_deref(), Some("<p>The post.</p>"),
"a long description is notes, not a subtitle");
assert_eq!(
body(Some("<p>The post.</p>"), Some(&long)).as_deref(),
Some("<p>The post.</p>"),
"a long description is notes, not a subtitle"
);
}
#[test]
@@ -1421,10 +1450,7 @@ mod tests {
<enclosure url="https://x/a.mp3" length="1" type="audio/mpeg"/></item>
</channel></rss>"#;
let feed = parse(xml).unwrap();
assert!(
feed.entries[0].explicit,
"the entry says nothing; the feed-level flag must still mark it explicit"
);
assert!(feed.entries[0].explicit, "the entry says nothing; the feed-level flag must still mark it explicit");
}
#[test]
@@ -1454,13 +1480,17 @@ mod tests {
explain_failure("the feed's address answered with nothing at all").unwrap().reason,
"This address answers with nothing; the site may be gone."
);
let dns = explain_failure("connecting: dns error: failed to lookup address information: Name or service not known").unwrap();
let dns =
explain_failure("connecting: dns error: failed to lookup address information: Name or service not known")
.unwrap();
assert_eq!(dns.reason, "This address no longer resolves; the site may be gone.");
// A lookup that failed for the moment is not a site gone (#143).
let blip = "connecting: dns error: failed to lookup address information: Temporary failure in name resolution";
assert!(is_temporary_lookup_failure(blip));
assert!(explain_failure(blip).is_none());
assert!(!is_temporary_lookup_failure("connecting: dns error: failed to lookup address information: Name or service not known"));
assert!(!is_temporary_lookup_failure(
"connecting: dns error: failed to lookup address information: Name or service not known"
));
let moved = explain_failure("got a web page, not a feed; it links https://x/feed as its feed").unwrap();
assert_eq!(moved.new_url.as_deref(), Some("https://x/feed"));
assert!(explain_failure("got a web page, not a feed").unwrap().new_url.is_none());
@@ -1671,10 +1701,7 @@ mod tests {
let two = &f.entries[0].enclosures;
assert_eq!(two.len(), 2, "both enclosures survive");
assert_eq!(
two[0].url, "https://x/a.mp3?v=1&t=2",
"document order, and the escaped ampersand is decoded"
);
assert_eq!(two[0].url, "https://x/a.mp3?v=1&t=2", "document order, and the escaped ampersand is decoded");
assert_eq!(two[0].length, Some(111));
assert_eq!(two[1].url, "https://x/b.mp4");
assert_eq!(two[1].mime.as_deref(), Some("video/mp4"));
@@ -1777,4 +1804,3 @@ mod tests {
assert_eq!(category_path("Software How-To"), ("Technology".into(), None));
}
}

27
crates/ipx/src/access.rs
View File

@@ -103,11 +103,8 @@ impl Keys {
/// Replaces the whole set, so a key Cloudflare has retired stops being accepted.
fn store(&self, set: JwkSet) {
let keys = set
.keys
.iter()
.filter_map(|k| Some((k.common.key_id.clone()?, DecodingKey::from_jwk(k).ok()?)))
.collect();
let keys =
set.keys.iter().filter_map(|k| Some((k.common.key_id.clone()?, DecodingKey::from_jwk(k).ok()?))).collect();
let mut c = self.cache.lock().unwrap();
c.keys = keys;
c.fetched = Some(Instant::now());
@@ -181,8 +178,7 @@ mod tests {
// HMAC and none, the classic ways to get a token past a verifier that trusts its header.
let hs = token(b"any secret at all", Algorithm::HS256, claims(AUD, 300));
assert_eq!(check(&keys, &hs).await, None, "HS256");
let none = format!("{}.{}.", "eyJhbGciOiJub25lIiwia2lkIjoiazEifQ",
ok.split('.').nth(1).unwrap());
let none = format!("{}.{}.", "eyJhbGciOiJub25lIiwia2lkIjoiazEifQ", ok.split('.').nth(1).unwrap());
assert_eq!(check(&keys, &none).await, None, "alg none");
}
@@ -191,16 +187,25 @@ mod tests {
let keys = Keys::default();
let ok = token(SIGNER, Algorithm::RS256, claims(AUD, 300));
let fetches = std::sync::atomic::AtomicUsize::new(0);
let count = || { fetches.fetch_add(1, std::sync::atomic::Ordering::SeqCst); async { Ok(jwks()) } };
assert_eq!(keys.verify_with(TEAM, AUD, &ok, count).await.as_deref(), Some("rays@sdf1.net"),
"a key not cached yet is fetched");
let count = || {
fetches.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async { Ok(jwks()) }
};
assert_eq!(
keys.verify_with(TEAM, AUD, &ok, count).await.as_deref(),
Some("rays@sdf1.net"),
"a key not cached yet is fetched"
);
assert_eq!(fetches.load(std::sync::atomic::Ordering::SeqCst), 1);
// A made-up key id straight after: not fetched again.
let mut h = Header::new(Algorithm::RS256);
h.kid = Some("nobody".into());
let stray = encode(&h, &claims(AUD, 300), &EncodingKey::from_rsa_der(SIGNER)).unwrap();
let count = || { fetches.fetch_add(1, std::sync::atomic::Ordering::SeqCst); async { Ok(jwks()) } };
let count = || {
fetches.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async { Ok(jwks()) }
};
assert_eq!(keys.verify_with(TEAM, AUD, &stray, count).await, None);
assert_eq!(fetches.load(std::sync::atomic::Ordering::SeqCst), 1, "rate-limited");
}

2
crates/ipx/src/art.rs
View File

@@ -1,8 +1,8 @@
//! Artwork kept on disk, so the page loads it from iPX: fast after the first time, nothing asked
//! of each publisher's server for every visit, and still there when a publisher is not.
use std::path::{Path, PathBuf};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{LazyLock, Mutex};
use std::time::{Duration, Instant};

4
crates/ipx/src/auth.rs
View File

@@ -25,9 +25,7 @@ pub fn verify_password(password: &str, stored: &str) -> bool {
tracing::warn!("stored password hash is unreadable; refusing the sign-in");
return false;
};
Argon2::default()
.verify_password(password.as_bytes(), &parsed)
.is_ok()
Argon2::default().verify_password(password.as_bytes(), &parsed).is_ok()
}
/// How many password checks run at once: each is tens of milliseconds of CPU, and forty wrong

63
crates/ipx/src/config.rs
View File

@@ -221,13 +221,7 @@ impl Default for General {
impl Default for Torrent {
fn default() -> Self {
Self {
enabled: true,
seed_ratio: 1.0,
seed_time_mins: 60,
port_range: "6881-6889".into(),
stall_mins: 30,
}
Self { enabled: true, seed_ratio: 1.0, seed_time_mins: 60, port_range: "6881-6889".into(), stall_mins: 30 }
}
}
@@ -309,14 +303,11 @@ impl Config {
// ponytail: a missing config is an empty one, so `list`/`add` work on a fresh install.
return Ok(Self::default());
}
let text = std::fs::read_to_string(path)
.with_context(|| format!("reading {}", path.display()))?;
let mut cfg: Self =
toml::from_str(&text).with_context(|| format!("parsing {}", path.display()))?;
let text = std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
let mut cfg: Self = toml::from_str(&text).with_context(|| format!("parsing {}", path.display()))?;
cfg.general.download_dir = expand_tilde(&cfg.general.download_dir);
Ok(cfg)
}
}
/// What the database keeps of the configuration (issue #18): the server settings the admin page
@@ -515,10 +506,7 @@ pub fn unique_slug(text: &str, taken: &BTreeMap<String, Feed>) -> String {
/// `$var`, or `~/fallback` when it is unset or empty, as the XDG base directory spec says.
fn xdg(var: &str, fallback: &str) -> PathBuf {
std::env::var_os(var)
.filter(|v| !v.is_empty())
.map(PathBuf::from)
.unwrap_or_else(|| home().join(fallback))
std::env::var_os(var).filter(|v| !v.is_empty()).map(PathBuf::from).unwrap_or_else(|| home().join(fallback))
}
fn home() -> PathBuf {
@@ -597,10 +585,20 @@ mod tests {
#[test]
fn intervals_parse_from_the_forms_people_actually_type() {
for (input, want) in [
("every 30m", 30), ("30m", 30), ("30", 30), ("every 30 minutes", 30),
("every 4h", 240), ("4h", 240), ("4 hours", 240), ("EVERY 4H", 240),
("1d", 1440), ("every 2 days", 2880), (" every 90m ", 90),
("1w", 10080), ("every 2 weeks", 20160), ("2 w", 20160),
("every 30m", 30),
("30m", 30),
("30", 30),
("every 30 minutes", 30),
("every 4h", 240),
("4h", 240),
("4 hours", 240),
("EVERY 4H", 240),
("1d", 1440),
("every 2 days", 2880),
(" every 90m ", 90),
("1w", 10080),
("every 2 weeks", 20160),
("2 w", 20160),
] {
assert_eq!(parse_interval(input), Some(want), "{input:?}");
}
@@ -672,11 +670,26 @@ mod tests {
assert!(slug(&"x".repeat(100)).len() <= 40);
let mut taken = BTreeMap::new();
taken.insert("the-daily".to_string(), Feed {
url: "u".into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
auto_download: true, max_new_per_check: None, username: None,
password: None, password_env: None, category: None, listed: false, private: false,
});
taken.insert(
"the-daily".to_string(),
Feed {
url: "u".into(),
folder: None,
group: None,
media_types: None,
schedule: None,
keywords: vec![],
allow_explicit: false,
auto_download: true,
max_new_per_check: None,
username: None,
password: None,
password_env: None,
category: None,
listed: false,
private: false,
},
);
assert_eq!(unique_slug("The Daily", &taken), "the-daily-2");
}

403
crates/ipx/src/db.rs
View File

@@ -1,8 +1,11 @@
//! The database, through SeaORM: SQLite today, Postgres to come (issue #18). Replaces the
//! per-feed .ipxd plists, history.dat and qmcache.dat.
use crate::entity::{
api_tokens, blocklists, catalogue, deadair_prefs, enclosures, entries, feeds, hidden, sessions, settings,
signal_verdicts, signals, subscriptions, users,
};
use anyhow::{Context, Result};
use crate::entity::{api_tokens, blocklists, catalogue, deadair_prefs, enclosures, entries, feeds, hidden, sessions, settings, signal_verdicts, signals, subscriptions, users};
use sea_orm::sea_query::{Expr, Func};
use sea_orm::{
ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, Set,
@@ -94,10 +97,9 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
// A feed is read whole only when it has changed since the last read, so each one's
// description would wait for its next item. Forgetting how each was last seen makes
// its next read, when it is due anyway, a whole one.
for sql in [
"ALTER TABLE feeds ADD COLUMN description TEXT",
"UPDATE feeds SET etag = NULL, last_modified = NULL",
] {
for sql in
["ALTER TABLE feeds ADD COLUMN description TEXT", "UPDATE feeds SET etag = NULL, last_modified = NULL"]
{
orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?;
}
}
@@ -125,7 +127,10 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
// Postgres guessed a third of entries were recent and read them all, entries_recent unused
// (#274). Analysed once when they are missing; autovacuum keeps them after.
if orm.get_database_backend() == sea_orm::DbBackend::Postgres {
let has = Statement::from_string(sea_orm::DbBackend::Postgres, "SELECT 1 FROM pg_stats WHERE tablename = 'entries_recent'");
let has = Statement::from_string(
sea_orm::DbBackend::Postgres,
"SELECT 1 FROM pg_stats WHERE tablename = 'entries_recent'",
);
if orm.query_one_raw(has).await?.is_none() {
orm.execute_unprepared("ANALYZE entries").await.context("analysing entries")?;
}
@@ -222,9 +227,7 @@ async fn connect(location: &str, keep_open: bool) -> Result<sea_orm::DatabaseCon
if keep_open && is_postgres(location) {
opts.min_connections(20);
}
sea_orm::Database::connect(opts)
.await
.with_context(|| format!("opening {}", redact(location)))
sea_orm::Database::connect(opts).await.with_context(|| format!("opening {}", redact(location)))
}
/// The URL `connect` hands sqlx for a location: a SQLite file, created if missing, or a Postgres
@@ -244,7 +247,6 @@ fn url_for(location: &str) -> String {
}
}
/// One person's wants for one feed. `None` in a field means the feed's own setting stands.
#[derive(Debug, Clone, Default)]
pub struct Sub {
@@ -330,7 +332,6 @@ pub struct Managed {
pub group_id: String,
}
/// What `ipx list` shows next to each configured feed.
#[derive(Debug, Default)]
pub struct FeedSummary {
@@ -414,9 +415,11 @@ impl Db {
admin.execute_unprepared(&format!("CREATE SCHEMA {schema}")).await?;
// Every connection in the pool starts in it, so each test sees only its own tables.
let sep = if url.contains('?') { '&' } else { '?' };
let orm =
connect(&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning"), false)
.await?;
let orm = connect(
&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning"),
false,
)
.await?;
create_missing(&orm).await?;
return Ok(Self { orm, tmp: None });
}
@@ -530,9 +533,9 @@ impl Db {
step("blocked"),
)
.await?
.into_iter()
.map(|b| (b.feed_id, keywords(Some(b.words)).unwrap_or_default()))
.collect();
.into_iter()
.map(|b| (b.feed_id, keywords(Some(b.words)).unwrap_or_default()))
.collect();
let mut rows = feeds::Entity::find();
if let Some(id) = only {
rows = rows.filter(feeds::Column::Id.eq(id));
@@ -574,7 +577,6 @@ pub struct FeedListing {
pub unread: i64,
}
/// Everything `fetch` needs to decide whether to poll a feed, and how.
#[derive(Debug, Default)]
pub struct HttpState {
@@ -611,7 +613,12 @@ impl Db {
/// Every feed's, in one query: a scan, and the daemon working out when the next is due, asked
/// feed by feed, 180 queries a minute (#114).
pub async fn http_states(&self) -> Result<std::collections::HashMap<String, HttpState>> {
Ok(feeds::Entity::find().all(&self.orm).await?.into_iter().map(|f| (f.id.clone(), HttpState::from(f))).collect())
Ok(feeds::Entity::find()
.all(&self.orm)
.await?
.into_iter()
.map(|f| (f.id.clone(), HttpState::from(f)))
.collect())
}
/// Upsert after a successful poll. Clears any previous error.
@@ -786,30 +793,18 @@ impl Db {
Ok(())
}
pub async fn record_enclosure(
&self,
feed_id: &str,
guid: &str,
enc: &crate::feed::Enclosure,
) -> Result<bool> {
pub async fn record_enclosure(&self, feed_id: &str, guid: &str, enc: &crate::feed::Enclosure) -> Result<bool> {
let inserted = self
.exec(
"INSERT INTO enclosures (feed_id, guid, url, mime, length, state)
VALUES ($1, $2, $3, $4, $5, 'pending') ON CONFLICT DO NOTHING",
vec![
feed_id.into(),
guid.into(),
enc.url.clone().into(),
enc.mime.clone().into(),
enc.length.into(),
],
vec![feed_id.into(), guid.into(), enc.url.clone().into(), enc.mime.clone().into(), enc.length.into()],
)
.await?;
Ok(inserted == 1)
}
}
/// An enclosure waiting to be downloaded.
#[derive(Debug)]
pub struct Pending {
@@ -848,7 +843,9 @@ impl Db {
.await?;
// And back, when a limit is raised or a feed downloads again.
self.exec(
&format!("UPDATE enclosures SET state = 'pending' WHERE feed_id = $1 AND state = 'held' AND guid IN ({newest})"),
&format!(
"UPDATE enclosures SET state = 'pending' WHERE feed_id = $1 AND state = 'held' AND guid IN ({newest})"
),
vec![feed_id.into(), limit],
)
.await?;
@@ -881,7 +878,6 @@ impl Db {
}
}
/// A downloaded file the reaper may consider.
#[derive(Debug, Clone, PartialEq)]
pub struct Candidate {
@@ -1019,7 +1015,9 @@ impl Db {
for table in ["entries", "feeds"] {
let col = if table == "feeds" { "id" } else { "feed_id" };
self.exec(
&format!("UPDATE {table} SET image = 'https://' || substr(image, 8) WHERE {col} = $1 AND image LIKE $2"),
&format!(
"UPDATE {table} SET image = 'https://' || substr(image, 8) WHERE {col} = $1 AND image LIKE $2"
),
vec![feed_id.into(), like.clone().into()],
)
.await?;
@@ -1108,19 +1106,14 @@ impl Db {
/// (pending, downloaded) across all feeds, for the status command.
pub async fn counts(&self) -> Result<(i64, i64)> {
let pending = enclosures::Entity::find()
.filter(enclosures::Column::State.eq("pending"))
.count(&self.orm)
.await?;
let downloaded = enclosures::Entity::find()
.filter(enclosures::Column::Path.is_not_null())
.count(&self.orm)
.await?;
let pending =
enclosures::Entity::find().filter(enclosures::Column::State.eq("pending")).count(&self.orm).await?;
let downloaded =
enclosures::Entity::find().filter(enclosures::Column::Path.is_not_null()).count(&self.orm).await?;
Ok((pending as i64, downloaded as i64))
}
}
/// An entry plus its enclosures, for the web UI.
#[derive(Debug, serde::Serialize)]
pub struct EntryRow {
@@ -1182,9 +1175,7 @@ fn entries_from(
let search = match search.map(|q| q.trim().to_lowercase()).filter(|q| !q.is_empty()) {
Some(q) => {
let like = a.p(format!("%{q}%"));
format!(
"(lower(coalesce(e.title, '')) LIKE {like} OR lower(coalesce(e.description, '')) LIKE {like})"
)
format!("(lower(coalesce(e.title, '')) LIKE {like} OR lower(coalesce(e.description, '')) LIKE {like})")
}
None => "true".into(),
};
@@ -1340,7 +1331,8 @@ impl Db {
pub async fn signal_count(&self, user_id: i64, signal: i64, filter: Filter, search: Option<&str>) -> Result<i64> {
let mut a = Args::default();
let sql = format!("SELECT count(*) AS n {}", entries_from(&mut a, user_id, None, filter, search, None, Some(signal)));
let sql =
format!("SELECT count(*) AS n {}", entries_from(&mut a, user_id, None, filter, search, None, Some(signal)));
let r = self.rows(&sql, a.0).await?;
Ok(r.first().context("count(*) always returns a row")?.try_get("", "n")?)
}
@@ -1432,7 +1424,8 @@ impl Db {
search: Option<&str>,
) -> Result<i64> {
let mut a = Args::default();
let sql = format!("SELECT count(*) AS n {}", entries_from(&mut a, user_id, feed_id, filter, search, None, None));
let sql =
format!("SELECT count(*) AS n {}", entries_from(&mut a, user_id, feed_id, filter, search, None, None));
let r = self.rows(&sql, a.0).await?;
Ok(r.first().context("count(*) always returns a row")?.try_get("", "n")?)
}
@@ -1457,13 +1450,7 @@ impl Db {
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (user_id, feed_id, guid) DO UPDATE SET position = excluded.position,
duration = coalesce(excluded.duration, entry_state.duration)",
vec![
user_id.into(),
feed_id.into(),
guid.into(),
secs.max(0).into(),
duration.filter(|d| *d > 0).into(),
],
vec![user_id.into(), feed_id.into(), guid.into(), secs.max(0).into(), duration.filter(|d| *d > 0).into()],
)
.await?;
Ok(())
@@ -1508,9 +1495,7 @@ impl Db {
return Ok(false); // dropped, so rolled back
}
for (id, feed) in &cfg.feeds {
catalogue::ActiveModel { id: Set(id.clone()), spec: Set(serde_json::to_string(feed)?) }
.insert(&tx)
.await?;
catalogue::ActiveModel { id: Set(id.clone()), spec: Set(serde_json::to_string(feed)?) }.insert(&tx).await?;
}
tx.commit().await?;
Ok(true)
@@ -1531,9 +1516,7 @@ impl Db {
.await?;
catalogue::Entity::delete_many().exec(&tx).await?;
for (id, feed) in &cfg.feeds {
catalogue::ActiveModel { id: Set(id.clone()), spec: Set(serde_json::to_string(feed)?) }
.insert(&tx)
.await?;
catalogue::ActiveModel { id: Set(id.clone()), spec: Set(serde_json::to_string(feed)?) }.insert(&tx).await?;
}
tx.commit().await?;
Ok(())
@@ -1583,10 +1566,7 @@ impl Db {
/// What this person wants from a feed. Absent means they do not subscribe at all.
pub async fn subscription(&self, user_id: i64, feed_id: &str) -> Result<Option<Sub>> {
Ok(subscriptions::Entity::find_by_id((user_id, feed_id.to_owned()))
.one(&self.orm)
.await?
.map(Sub::from))
Ok(subscriptions::Entity::find_by_id((user_id, feed_id.to_owned())).one(&self.orm).await?.map(Sub::from))
}
/// Every feed this person subscribes to, with their settings.
@@ -1651,14 +1631,11 @@ impl Db {
/// Feeds with any audio or video enclosure: the Directory's Podcasts, with the rest Blogs.
/// Reaped files keep their rows, so a show whose files have all been purged still counts.
pub async fn media_feeds(&self) -> Result<std::collections::HashSet<String>> {
self.rows(
"SELECT DISTINCT feed_id FROM enclosures WHERE mime LIKE 'audio/%' OR mime LIKE 'video/%'",
vec![],
)
.await?
.iter()
.map(|r| Ok(r.try_get("", "feed_id")?))
.collect()
self.rows("SELECT DISTINCT feed_id FROM enclosures WHERE mime LIKE 'audio/%' OR mime LIKE 'video/%'", vec![])
.await?
.iter()
.map(|r| Ok(r.try_get("", "feed_id")?))
.collect()
}
/// Who else would miss this file: subscribers other than `user_id` who have starred
@@ -1769,14 +1746,13 @@ impl Db {
.iter()
.flat_map(|e| {
let hay = [e.title.as_deref().unwrap_or(""), e.description.as_deref().unwrap_or("")];
lists
.iter()
.filter(move |(_, words)| crate::download::blocked(words, &hay))
.map(|(user, _)| hidden::ActiveModel {
lists.iter().filter(move |(_, words)| crate::download::blocked(words, &hay)).map(|(user, _)| {
hidden::ActiveModel {
user_id: Set(*user),
feed_id: Set(feed_id.to_owned()),
guid: Set(e.guid.clone()),
})
}
})
})
.collect();
if !rows.is_empty() {
@@ -1925,17 +1901,12 @@ impl Db {
/// Sets some of one person's columns, whatever else is in their row.
async fn update_user(&self, id: i64, m: users::ActiveModel) -> Result<()> {
users::Entity::update_many()
.set(m)
.filter(users::Column::Id.eq(id))
.exec(&self.orm)
.await?;
users::Entity::update_many().set(m).filter(users::Column::Id.eq(id)).exec(&self.orm).await?;
Ok(())
}
pub async fn set_password(&self, id: i64, hash: &str) -> Result<()> {
self.update_user(id, users::ActiveModel { pass_hash: Set(Some(hash.to_owned())), ..Default::default() })
.await
self.update_user(id, users::ActiveModel { pass_hash: Set(Some(hash.to_owned())), ..Default::default() }).await
}
pub async fn set_admin(&self, id: i64, admin: bool) -> Result<()> {
@@ -2233,11 +2204,8 @@ impl Db {
#[tracing::instrument(skip_all)]
pub async fn adopt(&self, parent: &str, child: &str, listed: &[(&str, &str)]) -> Result<()> {
use sea_orm::TransactionTrait;
let holds = enclosures::Entity::find()
.filter(enclosures::Column::FeedId.eq(parent))
.count(&self.orm)
.await?
> 0;
let holds =
enclosures::Entity::find().filter(enclosures::Column::FeedId.eq(parent)).count(&self.orm).await? > 0;
if !holds {
return Ok(()); // An OPML, or a creator already shared out.
}
@@ -2432,7 +2400,12 @@ impl Db {
.await?
.iter()
.map(|r| {
Ok((r.try_get("", "user_id")?, r.try_get("", "feed_id")?, r.try_get("", "wanted")?, r.try_get("", "pause_ms")?))
Ok((
r.try_get("", "user_id")?,
r.try_get("", "feed_id")?,
r.try_get("", "wanted")?,
r.try_get("", "pause_ms")?,
))
})
.collect()
}
@@ -2646,7 +2619,6 @@ pub struct Unjudged {
pub signals: Vec<Ask>,
}
impl Db {
pub async fn signals(&self, user_id: i64) -> Result<Vec<SignalRow>> {
self.signal_rows(user_id, None).await
@@ -2695,7 +2667,13 @@ impl Db {
.collect()
}
pub async fn add_signal(&self, user_id: i64, name: &str, words: &[String], description: Option<&str>) -> Result<i64> {
pub async fn add_signal(
&self,
user_id: i64,
name: &str,
words: &[String],
description: Option<&str>,
) -> Result<i64> {
let m = signals::ActiveModel {
user_id: Set(user_id),
name: Set(name.to_owned()),
@@ -2760,8 +2738,11 @@ impl Db {
}
pub async fn set_typesafe_key(&self, user_id: i64, key: Option<&str>) -> Result<()> {
self.update_user(user_id, users::ActiveModel { typesafe_key: Set(key.map(str::to_owned)), ..Default::default() })
.await
self.update_user(
user_id,
users::ActiveModel { typesafe_key: Set(key.map(str::to_owned)), ..Default::default() },
)
.await
}
/// Up to `limit` items, newest first, that someone with a key has a Signal to ask about: not
@@ -2804,7 +2785,9 @@ impl Db {
"JOIN signal_verdicts v ON v.signal_id = g.id AND v.feed_id = s.feed_id AND v.recheck
JOIN entries e ON e.feed_id = v.feed_id AND e.guid = v.guid",
);
let sql = format!("SELECT * FROM ({recent} UNION ALL {again}) q ORDER BY at DESC, feed_id, guid, user_id LIMIT {limit}");
let sql = format!(
"SELECT * FROM ({recent} UNION ALL {again}) q ORDER BY at DESC, feed_id, guid, user_id LIMIT {limit}"
);
let mut out: Vec<Unjudged> = Vec::new();
for r in self.rows(&sql, a.0).await? {
let (user_id, feed_id, guid): (i64, String, String) =
@@ -2853,10 +2836,7 @@ impl Db {
}
pub fn now() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)
}
#[cfg(test)]
@@ -2889,9 +2869,15 @@ mod tests {
db.store_plans(1, r#"{"300": {"cuts": [[1000,3000]], "pieces": [[0,10,1.0]]}}"#).await.unwrap();
assert!(db.deadair_todo().await.unwrap().is_empty());
let p = db.plans(&[1, 2]).await.unwrap();
assert_eq!((p.len(), p[&1][&300].cuts.as_slice(), p[&1][&300].saved_ms()), (1, &[crate::deadair::Cut(1000, 3000)][..], 2000));
assert_eq!(
(p.len(), p[&1][&300].cuts.as_slice(), p[&1][&300].saved_ms()),
(1, &[crate::deadair::Cut(1000, 3000)][..], 2000)
);
// Downloaded again since: looked at again before it plays shortened.
db.orm.execute_unprepared(&format!("UPDATE enclosures SET downloaded_at = {} WHERE id = 1", now() + 60)).await.unwrap();
db.orm
.execute_unprepared(&format!("UPDATE enclosures SET downloaded_at = {} WHERE id = 1", now() + 60))
.await
.unwrap();
assert_eq!(db.deadair_todo().await.unwrap().len(), 1);
assert!(db.plans(&[1]).await.unwrap().is_empty());
@@ -2929,7 +2915,9 @@ mod tests {
// And writers at once all get their turn, not a refusal.
let writes = (0..16).map(|i| {
let db = &db;
async move { db.record_feed(&format!("f{i}"), "http://x/f", Some("F"), None, None, None, None, None, None).await }
async move {
db.record_feed(&format!("f{i}"), "http://x/f", Some("F"), None, None, None, None, None, None).await
}
});
for r in futures_util::future::join_all(writes).await {
r.unwrap();
@@ -2942,10 +2930,18 @@ mod tests {
let backend = db.orm.get_database_backend();
if backend == sea_orm::DbBackend::Sqlite {
let row = db.orm.query_one_raw(Statement::from_string(backend, "PRAGMA synchronous")).await.unwrap();
assert_eq!(row.unwrap().try_get_by_index::<i32>(0).unwrap(), 1, "NORMAL, on every connection in the pool (#135)");
assert_eq!(
row.unwrap().try_get_by_index::<i32>(0).unwrap(),
1,
"NORMAL, on every connection in the pool (#135)"
);
} else {
let row = db.orm.query_one_raw(Statement::from_string(backend, "SHOW synchronous_commit")).await.unwrap();
assert_eq!(row.unwrap().try_get_by_index::<String>(0).unwrap(), "off", "with the test's own search_path as well (#140)");
assert_eq!(
row.unwrap().try_get_by_index::<String>(0).unwrap(),
"off",
"with the test's own search_path as well (#140)"
);
}
}
@@ -2954,7 +2950,11 @@ mod tests {
let db = Db::memory().await.unwrap();
let record = |etag: &'static str, about: Option<&'static str>| {
let db = &db;
async move { db.record_feed("f", "https://x/f", Some("F"), Some(etag), Some("then"), None, None, None, about).await.unwrap() }
async move {
db.record_feed("f", "https://x/f", Some("F"), Some(etag), Some("then"), None, None, None, about)
.await
.unwrap()
}
};
record("\"a\"", Some("About F")).await;
assert_eq!(db.feed_description("f").await.unwrap().as_deref(), Some("About F"));
@@ -2965,7 +2965,10 @@ mod tests {
create_missing(&db.orm).await.unwrap();
assert_eq!(db.feed_description("f").await.unwrap(), None);
let st = db.http_state("f").await.unwrap();
assert!(st.etag.is_none() && st.last_modified.is_none(), "every feed is read whole next time, for its description");
assert!(
st.etag.is_none() && st.last_modified.is_none(),
"every feed is read whole next time, for its description"
);
// With the column there, opening the database again writes nothing.
record("\"b\"", Some("About F")).await;
create_missing(&db.orm).await.unwrap();
@@ -3001,10 +3004,12 @@ mod tests {
INSERT INTO enclosures (id, feed_id, guid, url, mime, length, state) VALUES
(1,'f','a','u1','audio/mpeg',300,'pending'),(2,'g','b','u2','image/png',10,'pending'),
(3,'f','c','u3','video/mp4',2000,'pending');",
).await
)
.await
.unwrap();
let order = async |col: &str, dir: &str| -> Vec<String> {
db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir)).await
db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir))
.await
.unwrap()
.into_iter()
.map(|e| e.guid)
@@ -3025,8 +3030,13 @@ mod tests {
db.exec_for_test("DELETE FROM entries WHERE guid = 'n';").await.unwrap();
assert_eq!(order("published", "desc").await, ["c", "b", "a"]);
// A page from the middle: the keys are paged before the rows are read (#188).
let page: Vec<String> = db.entries_in(1, None, Filter::All, None, 1, 1, &order_sql("published", "desc")).await
.unwrap().into_iter().map(|e| e.guid).collect();
let page: Vec<String> = db
.entries_in(1, None, Filter::All, None, 1, 1, &order_sql("published", "desc"))
.await
.unwrap()
.into_iter()
.map(|e| e.guid)
.collect();
assert_eq!(page, ["b"]);
db.set_entry_flag(1, "f", "a", EntryFlag::Flagged, true).await.unwrap();
assert_eq!(order("kept", "desc").await[0], "a");
@@ -3055,7 +3065,10 @@ mod tests {
let list = db.feed_list(1, None).await.unwrap();
let g = db.feed_list(1, Some("g")).await.unwrap();
assert_eq!(g.keys().collect::<Vec<_>>(), ["g"]);
assert_eq!((g["g"].unread, g["g"].summary.entries, g["g"].blocked.clone()), (1, 1, vec!["spoiler".to_string()]));
assert_eq!(
(g["g"].unread, g["g"].summary.entries, g["g"].blocked.clone()),
(1, 1, vec!["spoiler".to_string()])
);
for id in ["f", "g"] {
let one = &list[id];
let s = db.feed_summary(id).await.unwrap();
@@ -3084,8 +3097,8 @@ mod tests {
db.secure_images("f", "a.example").await.unwrap();
assert_eq!(db.feed_summary("f").await.unwrap().image.as_deref(), Some("https://a.example/logo.png"));
assert!(db.names_image("https://a.example/1.jpg").await.unwrap());
assert!(db.names_image("http://b.example/2.jpg").await.unwrap()); // another host
assert!(db.names_image("http://a.example/3.jpg").await.unwrap()); // another feed
assert!(db.names_image("http://b.example/2.jpg").await.unwrap()); // another host
assert!(db.names_image("http://a.example/3.jpg").await.unwrap()); // another feed
assert_eq!(db.feed_summary("g").await.unwrap().image.as_deref(), Some("http://a.example/g.png"));
}
@@ -3102,7 +3115,8 @@ mod tests {
INSERT INTO entries (feed_id, guid, first_seen, published) VALUES
('quiet','q',0,100),('news','n',0,950),('dead','d',0,950),('kept','k',0,100);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'kept','k','u','/tmp/k','done');",
).await
)
.await
.unwrap();
// Dead before 500, or nothing newer than 500: dead fails since 100, quiet's newest is 100.
// news published lately, new has no items yet, wanted has a subscriber, kept a file,
@@ -3128,7 +3142,8 @@ mod tests {
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('gone','a',0),('kept','b',0);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'gone','a','u1',null,'pending'),(2,'kept','b','u2','/tmp/b','done');",
).await
)
.await
.unwrap();
// Subscribed, holding a file, working, or in the catalogue: all stay.
assert_eq!(db.prune_abandoned_failures().await.unwrap(), vec!["gone".to_string()]);
@@ -3147,21 +3162,27 @@ mod tests {
INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES
(1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null),
(3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','pending',null);",
).await
)
.await
.unwrap();
let states = || async {
let mut v: Vec<(i64, String)> = db.rows("SELECT id, state FROM enclosures ORDER BY id", vec![]).await.unwrap()
.iter().map(|r| (r.try_get("", "id").unwrap(), r.try_get("", "state").unwrap())).collect();
let mut v: Vec<(i64, String)> = db
.rows("SELECT id, state FROM enclosures ORDER BY id", vec![])
.await
.unwrap()
.iter()
.map(|r| (r.try_get("", "id").unwrap(), r.try_get("", "state").unwrap()))
.collect();
v.sort();
v.into_iter().map(|(_, s)| s).collect::<Vec<_>>()
};
db.hold_back("f", 2).await.unwrap(); // newest two: e4, e3
db.hold_back("f", 2).await.unwrap(); // newest two: e4, e3
assert_eq!(states().await, ["held", "held", "done", "pending"]);
db.hold_back("f", 3).await.unwrap(); // a limit raised brings e2 back
db.hold_back("f", 3).await.unwrap(); // a limit raised brings e2 back
assert_eq!(states().await, ["held", "pending", "done", "pending"]);
db.hold_back("f", 0).await.unwrap(); // a feed nobody downloads: all held
db.hold_back("f", 0).await.unwrap(); // a feed nobody downloads: all held
assert_eq!(states().await, ["held", "held", "done", "held"]);
db.hold_back("f", usize::MAX).await.unwrap(); // unlimited, an archive: all queued
db.hold_back("f", usize::MAX).await.unwrap(); // unlimited, an archive: all queued
assert_eq!(states().await, ["pending", "pending", "done", "pending"]);
}
@@ -3175,7 +3196,8 @@ mod tests {
(1,'f','e1','u1','pending',null),(2,'f','e2','u2','pending',null),
(3,'f','e3','u3','done','/tmp/3'),(4,'f','e4','u4','done','/tmp/4'),
(5,'f','e5','u5','skipped',null);",
).await
)
.await
.unwrap();
let ids = |p: Vec<Pending>| p.into_iter().map(|p| p.id).collect::<Vec<_>>();
// The newest three with a file worth having are e4, e3 and e2 (e5's was skipped): only
@@ -3225,12 +3247,15 @@ mod tests {
db.record_entry("f", &item("b", "B", Some("the next review"))).await.unwrap();
db.record_entry("f", &item("c", "C", Some("as it was"))).await.unwrap();
let changed = db
.changed_items("f", &[
item("a", "A", Some("the review, fixed")), // the text corrected
item("b", "B renamed", Some("the next review")), // retitled
item("c", "C", None), // says nothing of its text: what is stored stands
item("d", "D", Some("new")), // not stored: the insert's, not this
])
.changed_items(
"f",
&[
item("a", "A", Some("the review, fixed")), // the text corrected
item("b", "B renamed", Some("the next review")), // retitled
item("c", "C", None), // says nothing of its text: what is stored stands
item("d", "D", Some("new")), // not stored: the insert's, not this
],
)
.await
.unwrap();
assert_eq!(changed, ["a", "b"].map(String::from).into());
@@ -3243,7 +3268,8 @@ mod tests {
db.exec_for_test(
"INSERT INTO feeds (id, url, image) VALUES ('f','u','http://cdn.example/show.jpg');
INSERT INTO entries (feed_id, guid, first_seen, image) VALUES ('f','a',0,'http://cdn.example/ep.jpg');",
).await
)
.await
.unwrap();
assert!(db.names_image("http://cdn.example/show.jpg").await.unwrap());
assert!(db.names_image("http://cdn.example/ep.jpg").await.unwrap());
@@ -3259,7 +3285,8 @@ mod tests {
INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1,'f','a','u1','/tmp/a','done');",
).await
)
.await
.unwrap();
// Nobody has touched it: both others still have it unplayed.
@@ -3283,7 +3310,8 @@ mod tests {
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO entries (feed_id, guid, title, first_seen) VALUES
('f','a','One',100),('f','b','Two',200);",
).await
)
.await
.unwrap();
assert_eq!(db.unread_count(1, "f").await.unwrap(), 2);
@@ -3362,8 +3390,7 @@ mod tests {
// Cutting this along with the dead read columns left the browser suite's admin with an
// empty sidebar: it is how a fresh install's first account gets config.toml's feeds.
let db = Db::memory().await.unwrap();
db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',true);").await
.unwrap();
db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',true);").await.unwrap();
let subs = async || db.i64s_for_test("SELECT count(*) FROM subscriptions").await[0];
let catalogue = ["a".to_string(), "b".to_string()];
assert_eq!(db.adopt_catalogue(1, &catalogue).await.unwrap(), 2);
@@ -3405,7 +3432,8 @@ mod tests {
-- Opened, and so read, but 42 of 900 seconds in: still Currently Listening.
-- Filtering on read hid exactly these (issue #14).
(7,'f','h',true,false,42);",
).await
)
.await
.unwrap();
for f in [Filter::All, Filter::Unread, Filter::Downloaded, Filter::Flagged, Filter::InProgress] {
@@ -3425,8 +3453,11 @@ mod tests {
assert_eq!(db.count_in(7, Some("f"), Filter::Downloaded, None).await.unwrap(), 1);
assert_eq!(db.count_in(7, Some("f"), Filter::Flagged, None).await.unwrap(), 1);
assert_eq!(db.count_in(7, Some("f"), Filter::All, Some("dive")).await.unwrap(), 2);
assert_eq!(db.count_in(7, Some("f"), Filter::All, Some("NOTES two")).await.unwrap(), 1,
"search is case-insensitive and covers the description");
assert_eq!(
db.count_in(7, Some("f"), Filter::All, Some("NOTES two")).await.unwrap(),
1,
"search is case-insensitive and covers the description"
);
// Currently Listening: started, not finished, and not just an accidental tap.
let order = order_sql("published", "desc");
@@ -3473,14 +3504,26 @@ mod tests {
let everyone = db.unjudged(&[], 50, true, 30).await.unwrap();
let mut seen: Vec<_> = everyone.iter().map(|u| (u.user_id, u.guid.as_str(), u.key.is_none())).collect();
seen.sort();
assert_eq!(seen, [(1, "x", true), (1, "y", true), (2, "x", true)], "on the server's key, everyone's Signals, no key of their own needed");
assert_eq!(
seen,
[(1, "x", true), (1, "y", true), (2, "x", true)],
"on the server's key, everyone's Signals, no key of their own needed"
);
db.set_typesafe_key(1, Some("k")).await.unwrap();
let queue = db.unjudged(&[], 50, false, 30).await.unwrap();
let mut seen: Vec<_> = queue.iter().map(|u| (u.guid.as_str(), u.signals.len())).collect();
seen.sort();
assert_eq!(seen, [("x", 2), ("y", 2)], "each recent subscribed item once, with both Signals; old and unsubscribed left out");
assert_eq!(
seen,
[("x", 2), ("y", 2)],
"each recent subscribed item once, with both Signals; old and unsubscribed left out"
);
let ask = queue[0].signals.iter().find(|a| a.id == food).unwrap();
assert_eq!((ask.name.as_str(), ask.description.as_deref()), ("Food", Some("Bread and cakes")), "Jev is asked about the name and the sentence");
assert_eq!(
(ask.name.as_str(), ask.description.as_deref()),
("Food", Some("Bread and cakes")),
"Jev is asked about the name and the sentence"
);
assert!(db.unjudged(&[1], 50, false, 30).await.unwrap().is_empty(), "a refused key is passed over");
db.record_verdicts("f", "x", &[(f1, true), (food, false)]).await.unwrap();
@@ -3491,10 +3534,15 @@ mod tests {
let page = db.signal_entries(1, f1, Filter::All, None, 0, 50, &order).await.unwrap();
assert_eq!(page.iter().map(|e| e.guid.as_str()).collect::<Vec<_>>(), ["x"]);
assert_eq!(db.signal_count(1, food, Filter::All, None).await.unwrap(), 1);
assert_eq!(db.signals(1).await.unwrap().iter().map(|s| (s.items, s.unread)).collect::<Vec<_>>(), [(1, 1), (1, 1)]);
db.exec_for_test("INSERT INTO entry_state (user_id, feed_id, guid, read, flagged, position) VALUES (1,'f','x',true,false,0)")
.await
.unwrap();
assert_eq!(
db.signals(1).await.unwrap().iter().map(|s| (s.items, s.unread)).collect::<Vec<_>>(),
[(1, 1), (1, 1)]
);
db.exec_for_test(
"INSERT INTO entry_state (user_id, feed_id, guid, read, flagged, position) VALUES (1,'f','x',true,false,0)",
)
.await
.unwrap();
let f = db.signal(1, f1).await.unwrap().unwrap();
assert_eq!((f.items, f.unread), (1, 0), "read in its feed is read in the Signal, and still one of its items");
@@ -3505,11 +3553,22 @@ mod tests {
// An edit asks again only what was picked out, however old, and leaves what was passed
// over (#276). Until answered, what was picked out stays in the list.
db.record_verdicts("g", "old", &[(f1, true)]).await.unwrap();
db.set_signal(1, f1, SignalChange { description: Some(Some("Grand prix racing")), ..Default::default() }).await.unwrap();
db.set_signal(1, f1, SignalChange { description: Some(Some("Grand prix racing")), ..Default::default() })
.await
.unwrap();
assert_eq!(db.signal(1, f1).await.unwrap().unwrap().description.as_deref(), Some("Grand prix racing"));
let again: Vec<_> = db.unjudged(&[], 50, false, 30).await.unwrap().iter()
.map(|u| (u.guid.clone(), u.signals.iter().map(|a| a.id).collect::<Vec<_>>())).collect();
assert_eq!(again, [("x".to_string(), vec![f1]), ("old".to_string(), vec![f1])], "y was passed over, and stays so");
let again: Vec<_> = db
.unjudged(&[], 50, false, 30)
.await
.unwrap()
.iter()
.map(|u| (u.guid.clone(), u.signals.iter().map(|a| a.id).collect::<Vec<_>>()))
.collect();
assert_eq!(
again,
[("x".to_string(), vec![f1]), ("old".to_string(), vec![f1])],
"y was passed over, and stays so"
);
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 2);
db.record_verdicts("f", "x", &[(f1, false)]).await.unwrap();
db.record_verdicts("g", "old", &[(f1, true)]).await.unwrap();
@@ -3518,7 +3577,9 @@ mod tests {
assert!(db.unjudged(&[], 50, false, 1).await.unwrap().is_empty());
db.record_verdicts("f", "x", &[(f1, true)]).await.unwrap();
db.set_signal(1, f1, SignalChange { icon: Some(Some("solid/flag-checkered")), ..Default::default() }).await.unwrap();
db.set_signal(1, f1, SignalChange { icon: Some(Some("solid/flag-checkered")), ..Default::default() })
.await
.unwrap();
assert_eq!(db.signal_count(1, f1, Filter::All, None).await.unwrap(), 2, "x and old, kept");
assert!(db.unjudged(&[], 50, false, 30).await.unwrap().is_empty(), "a new icon asks nothing again");
assert_eq!(db.signal(1, f1).await.unwrap().unwrap().icon.as_deref(), Some("solid/flag-checkered"));
@@ -3574,7 +3635,8 @@ mod tests {
(1,'f','old','u-old','pending'),
(2,'f','mid','u-mid','pending'),
(3,'f','new','u-new','pending');",
).await
)
.await
.unwrap();
let got: Vec<String> = db.pending("f", 2).await.unwrap().into_iter().map(|p| p.url).collect();
assert_eq!(got, vec!["u-new", "u-mid"], "newest first, oldest left for later");
@@ -3589,7 +3651,8 @@ mod tests {
(2,'f','b','u2','pending',NULL),
(3,'f','c','u3','downloading','/tmp/already-here'),
(4,'f','d','u4','done','/tmp/x');",
).await
)
.await
.unwrap();
assert_eq!(db.requeue_interrupted().await.unwrap(), 1, "only the in-flight, fileless one");
let state = async |id: i64| db.strings_for_test(&format!("SELECT state FROM enclosures WHERE id = {id}")).await;
@@ -3609,10 +3672,12 @@ mod tests {
(3,'creator','c','u3','skipped',NULL,'explicit'),
(4,'other','d','u4','skipped',NULL,'torrents disabled');
INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',true);",
).await
)
.await
.unwrap();
db.adopt("creator", "show", &[("a", "u1"), ("b", "u2"), ("d", "u4")]).await.unwrap();
let owner = async |id: i64| db.strings_for_test(&format!("SELECT feed_id FROM enclosures WHERE id = {id}")).await;
let owner =
async |id: i64| db.strings_for_test(&format!("SELECT feed_id FROM enclosures WHERE id = {id}")).await;
assert_eq!(owner(1).await, ["show"], "a downloaded file moves with its item");
assert_eq!(owner(2).await, ["show"]);
assert_eq!(owner(3).await, ["creator"], "this show does not list it");
@@ -3637,7 +3702,8 @@ mod tests {
"INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false);
INSERT INTO subscriptions (user_id, feed_id, allow_explicit) VALUES
(1,'group',true),(1,'show',NULL),(2,'group',true),(2,'show',false);",
).await
)
.await
.unwrap();
let explicit = async |group| -> Vec<Option<bool>> {
let mut v: Vec<_> =
@@ -3645,7 +3711,11 @@ mod tests {
v.sort();
v
};
assert_eq!(explicit(Some("group")).await, [Some(false), Some(true)], "ray inherits; sam's own choice on the show wins");
assert_eq!(
explicit(Some("group")).await,
[Some(false), Some(true)],
"ray inherits; sam's own choice on the show wins"
);
assert_eq!(explicit(None).await, [None, Some(false)], "outside a group nothing is inherited");
}
@@ -3677,7 +3747,8 @@ mod tests {
db.subscribe(me, "f").await.unwrap();
assert!(db.set_pinned(me, "f", true).await.unwrap());
// set_subscription writes the rest of the row; it must leave the pin alone.
db.set_subscription(me, &Sub { feed_id: "f".into(), auto_download: Some(false), ..Default::default() }).await
db.set_subscription(me, &Sub { feed_id: "f".into(), auto_download: Some(false), ..Default::default() })
.await
.unwrap();
assert!(db.pinned_feeds(me).await.unwrap().contains("f"));
db.set_pinned(me, "f", false).await.unwrap();
@@ -3687,8 +3758,15 @@ mod tests {
#[test]
fn postgres_is_asked_for_no_notices_and_a_password_never_reaches_a_log() {
assert_eq!(url_for("postgres://u:p@h/d"), format!("postgres://u:p@h/d?{QUIET}"));
assert_eq!(url_for("postgres://u:p@h/d?sslmode=disable"), format!("postgres://u:p@h/d?sslmode=disable&{QUIET}"));
assert_eq!(url_for("postgres://u:p@h/d?options=-c%20x%3Dy"), "postgres://u:p@h/d?options=-c%20x%3Dy", "theirs stands");
assert_eq!(
url_for("postgres://u:p@h/d?sslmode=disable"),
format!("postgres://u:p@h/d?sslmode=disable&{QUIET}")
);
assert_eq!(
url_for("postgres://u:p@h/d?options=-c%20x%3Dy"),
"postgres://u:p@h/d?options=-c%20x%3Dy",
"theirs stands"
);
assert_eq!(url_for("/data/state.db"), "sqlite:///data/state.db?mode=rwc");
assert_eq!(redact("postgres://ipodderx:s3cret@h:5433/ipodderx"), "postgres://ipodderx:***@h:5433/ipodderx");
}
@@ -3732,7 +3810,8 @@ mod tests {
#[tokio::test]
async fn enclosure_url_is_the_dedupe_key() {
let db = Db::memory().await.unwrap();
let insert = "INSERT INTO enclosures (feed_id, guid, url, state) VALUES ('f', 'g', 'http://x/a.mp3', 'pending')";
let insert =
"INSERT INTO enclosures (feed_id, guid, url, state) VALUES ('f', 'g', 'http://x/a.mp3', 'pending')";
db.exec_for_test(insert).await.unwrap();
assert!(db.exec_for_test(insert).await.is_err(), "duplicate url must be rejected");
}

146
crates/ipx/src/deadair.rs
View File

@@ -164,8 +164,7 @@ pub fn pieces(frames: &[(u64, u64)], drop: &[(u64, u64)], frame: f64) -> Vec<(u6
}
let dropped = drops.peek().is_some_and(|&&(a, _)| a as usize <= n);
// Something between two frames (an ID3 tag partway through) ends a piece too.
let joins = !dropped
&& run.is_some_and(|(f, c)| c < most && frames[f + c - 1].0 + frames[f + c - 1].1 == pos);
let joins = !dropped && run.is_some_and(|(f, c)| c < most && frames[f + c - 1].0 + frames[f + c - 1].1 == pos);
if !joins && let Some(r) = run.take() {
push(&mut out, r);
}
@@ -261,9 +260,7 @@ impl Wanting {
}
pub fn wants(&self, user: i64, feed: &str) -> bool {
self.people
.get(&user)
.is_some_and(|p| wants(p, feed, self.groups.get(feed).map(String::as_str)))
self.people.get(&user).is_some_and(|p| wants(p, feed, self.groups.get(feed).map(String::as_str)))
}
/// The pause this person has DeadAir leave in a feed: the feed's, else its group's.
@@ -415,14 +412,15 @@ async fn next(ctx: &Ctx, failed: &mut std::collections::HashSet<i64>) -> Result<
let todo = ctx.db.deadair_todo().await?;
// For /metrics (#195): files of feeds someone has DeadAir on, from memory, without the
// per-file subscription and finished checks below.
let wanted = |t: &crate::db::DeadAirTodo| !failed.contains(&t.id) && wanting.people.keys().any(|&u| wanting.wants(u, &t.feed_id));
let wanted = |t: &crate::db::DeadAirTodo| {
!failed.contains(&t.id) && wanting.people.keys().any(|&u| wanting.wants(u, &t.feed_id))
};
st.deadair_backlog.store(todo.iter().filter(|t| wanted(t)).count() as i64, Relaxed);
for t in todo {
if failed.contains(&t.id) {
continue;
}
let users: Vec<i64> =
wanting.people.keys().copied().filter(|&u| wanting.wants(u, &t.feed_id)).collect();
let users: Vec<i64> = wanting.people.keys().copied().filter(|&u| wanting.wants(u, &t.feed_id)).collect();
if users.is_empty() || !ctx.db.subscribed_any(&users, &t.feed_id).await? {
continue;
}
@@ -446,11 +444,18 @@ async fn next(ctx: &Ctx, failed: &mut std::collections::HashSet<i64>) -> Result<
return Ok(true);
}
};
ctx.db.store_plans(t.id, &serde_json::to_string(&plans)?).instrument(span.in_scope(|| step("store_plans"))).await?;
ctx.db
.store_plans(t.id, &serde_json::to_string(&plans)?)
.instrument(span.in_scope(|| step("store_plans")))
.await?;
let usual = plans.get(&DEFAULT_PAUSE);
span.record("cuts", usual.map_or(0, |p| p.cuts.len()));
let saved_ms = usual.map_or(0, Plan::saved_ms);
ctx.out.emit(crate::ipc::Event::DeadairDone { feed: t.feed_id.clone(), enclosure: t.id, saved_s: saved_ms / 1000 });
ctx.out.emit(crate::ipc::Event::DeadairDone {
feed: t.feed_id.clone(),
enclosure: t.id,
saved_s: saved_ms / 1000,
});
// The event says it too (ev="deadair_done"); this line has how long it took and how many cuts.
tracing::info!(
feed = t.feed_id,
@@ -472,8 +477,17 @@ pub async fn analyse(src: &Path) -> Result<Option<BTreeMap<i64, Plan>>> {
let Some((first, frame)) = probe(src).instrument(step("probe")).await? else { return Ok(None) };
let path = src.display().to_string();
let log = ffmpeg(&[
"-hide_banner", "-nostats", "-i", &path, "-map", "0:a:0",
"-af", &format!("silencedetect=n={THRESHOLD}:d={MIN_SILENCE}"), "-f", "null", "-",
"-hide_banner",
"-nostats",
"-i",
&path,
"-map",
"0:a:0",
"-af",
&format!("silencedetect=n={THRESHOLD}:d={MIN_SILENCE}"),
"-f",
"null",
"-",
])
.instrument(step("silencedetect"))
.await
@@ -492,16 +506,15 @@ pub async fn analyse(src: &Path) -> Result<Option<BTreeMap<i64, Plan>>> {
/// Where each audio frame is in the file, as (byte offset, bytes), in order.
async fn frame_bytes(path: &str) -> Result<Vec<(u64, u64)>> {
let out = run_out("ffprobe", &[
"-v", "error", "-select_streams", "a:0", "-show_entries", "packet=pos,size", "-of", "compact=p=0", path,
])
let out = run_out(
"ffprobe",
&["-v", "error", "-select_streams", "a:0", "-show_entries", "packet=pos,size", "-of", "compact=p=0", path],
)
.await?;
// ffprobe writes the fields in its own order, not the order asked for: by name, then.
out.lines()
.map(|l| {
let field = |k: &str| {
l.split('|').find_map(|f| f.strip_prefix(k)?.strip_prefix('=')?.parse::<u64>().ok())
};
let field = |k: &str| l.split('|').find_map(|f| f.strip_prefix(k)?.strip_prefix('=')?.parse::<u64>().ok());
field("pos").zip(field("size")).with_context(|| format!("reading ffprobe's frame: {l}"))
})
.collect()
@@ -511,26 +524,44 @@ async fn frame_bytes(path: &str) -> Result<Vec<(u64, u64)>> {
/// can shorten; None for one it cannot. A cover picture in an MP3 is not video.
async fn probe(src: &Path) -> Result<Option<(f64, f64)>> {
let path = src.display().to_string();
let out = run_out("ffprobe", &[
"-v", "error", "-show_entries",
"format=format_name:stream=codec_type,codec_name:stream_disposition=attached_pic", "-of", "json", &path,
])
let out = run_out(
"ffprobe",
&[
"-v",
"error",
"-show_entries",
"format=format_name:stream=codec_type,codec_name:stream_disposition=attached_pic",
"-of",
"json",
&path,
],
)
.await?;
let v: serde_json::Value = serde_json::from_str(&out).context("reading ffprobe")?;
let streams = v["streams"].as_array().cloned().unwrap_or_default();
let video = streams
.iter()
.any(|s| s["codec_type"] == "video" && s["disposition"]["attached_pic"] != 1);
let video = streams.iter().any(|s| s["codec_type"] == "video" && s["disposition"]["attached_pic"] != 1);
let audio = streams.iter().find(|s| s["codec_type"] == "audio");
let format = v["format"]["format_name"].as_str().unwrap_or("");
let plain = |codec: &str| format == codec && audio.is_some_and(|a| a["codec_name"] == codec);
if video || !(plain("mp3") || plain("aac")) {
return Ok(None);
}
let out = run_out("ffprobe", &[
"-v", "error", "-select_streams", "a:0", "-show_entries", "packet=pts_time,duration_time",
"-read_intervals", "%+#1", "-of", "csv=p=0", &path,
])
let out = run_out(
"ffprobe",
&[
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"packet=pts_time,duration_time",
"-read_intervals",
"%+#1",
"-of",
"csv=p=0",
&path,
],
)
.await?;
let mut f = out.trim().split(',').map(|x| x.trim().parse::<f64>());
match (f.next(), f.next()) {
@@ -682,9 +713,16 @@ mod tests {
for (ext, codec) in [("mp3", "libmp3lame"), ("aac", "aac"), ("m4a", "aac")] {
let src = dir.join(format!("t.{ext}"));
ffmpeg(&[
"-v", "error", "-y", "-f", "lavfi", "-i",
"-v",
"error",
"-y",
"-f",
"lavfi",
"-i",
"aevalsrc='if(lt(mod(t,3),1),0.5*sin(2*PI*440*t),0)':d=12:s=44100",
"-c:a", codec, &src.display().to_string(),
"-c:a",
codec,
&src.display().to_string(),
])
.await
.unwrap();
@@ -707,18 +745,36 @@ mod tests {
let len = |p: &Path| {
let p = p.display().to_string();
async move {
run_out("ffprobe", &["-v", "error", "-select_streams", "a:0", "-show_entries", "packet=duration_time", "-of", "csv=p=0", &p])
.await
.unwrap()
.lines()
.filter_map(|l| l.trim().trim_end_matches(',').parse::<f64>().ok())
.sum::<f64>()
run_out(
"ffprobe",
&[
"-v",
"error",
"-select_streams",
"a:0",
"-show_entries",
"packet=duration_time",
"-of",
"csv=p=0",
&p,
],
)
.await
.unwrap()
.lines()
.filter_map(|l| l.trim().trim_end_matches(',').parse::<f64>().ok())
.sum::<f64>()
}
};
let short = len(&src).await - len(&list).await;
assert!((short * 1000.0 - cut as f64).abs() < 100.0, "{ext}: {short} s shorter, {cut} ms cut");
// Decodes without a complaint.
assert!(ffmpeg(&["-v", "error", "-i", &list.display().to_string(), "-f", "null", "-"]).await.unwrap().is_empty());
assert!(
ffmpeg(&["-v", "error", "-i", &list.display().to_string(), "-f", "null", "-"])
.await
.unwrap()
.is_empty()
);
// Saved as a file of its own (#186): as long as the playlist, and it plays.
let (n, body) = kept(&src, p).await.unwrap();
let parts: Vec<_> = futures_util::TryStreamExt::try_collect(body).await.unwrap();
@@ -730,7 +786,12 @@ mod tests {
let saved = dir.join(format!("saved.{ext}"));
std::fs::write(&saved, &bytes).unwrap();
assert!((len(&saved).await - len(&list).await).abs() < 0.05, "{ext}");
assert!(ffmpeg(&["-v", "error", "-i", &saved.display().to_string(), "-f", "null", "-"]).await.unwrap().is_empty());
assert!(
ffmpeg(&["-v", "error", "-i", &saved.display().to_string(), "-f", "null", "-"])
.await
.unwrap()
.is_empty()
);
}
std::fs::remove_dir_all(&dir).unwrap();
}
@@ -750,6 +811,11 @@ async fn real_file() {
let name = src.file_name().unwrap().to_string_lossy().into_owned();
for (pause, p) in &plans {
std::fs::write(src.with_file_name(format!("{name}.{pause}.m3u8")), playlist(p, &name)).unwrap();
eprintln!("{pause} ms: {} cuts, {:.1} min saved, {} pieces", p.cuts.len(), p.saved_ms() as f64 / 60_000.0, p.pieces.len());
eprintln!(
"{pause} ms: {} cuts, {:.1} min saved, {} pieces",
p.cuts.len(),
p.saved_ms() as f64 / 60_000.0,
p.pieces.len()
);
}
}

41
crates/ipx/src/directory.rs
View File

@@ -26,8 +26,12 @@ pub enum DirError {
impl std::fmt::Display for DirError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DirError::NotConfigured => write!(f, "No directory is set up on this server. An admin sets one in Admin, Directory."),
DirError::Refused(401, _) => write!(f, "The directory does not accept this server's token. It may have been revoked."),
DirError::NotConfigured => {
write!(f, "No directory is set up on this server. An admin sets one in Admin, Directory.")
}
DirError::Refused(401, _) => {
write!(f, "The directory does not accept this server's token. It may have been revoked.")
}
DirError::Refused(403, msg) => write!(f, "The directory refused: {msg}."),
DirError::Refused(404, _) => write!(f, "That is not in the directory."),
DirError::Refused(code, msg) => write!(f, "The directory answered {code}: {msg}"),
@@ -58,9 +62,18 @@ pub fn configured(cfg: &config::Config) -> bool {
/// A span the whole call, its answer read and parsed included: the directory's own span ends when it
/// has answered, so a slow read of a large answer showed under no span (#250).
#[tracing::instrument(name = "directory_call", skip_all, fields(http.request.method = %method, url.path = %path.split('?').next().unwrap_or(path), http.response.body.size))]
async fn call<T: DeserializeOwned>(ctx: &Ctx, method: reqwest::Method, path: &str, body: Option<&[SnapshotEntry]>) -> Result<T, DirError> {
async fn call<T: DeserializeOwned>(
ctx: &Ctx,
method: reqwest::Method,
path: &str,
body: Option<&[SnapshotEntry]>,
) -> Result<T, DirError> {
let (base, token) = target(&ctx.cfg()).ok_or(DirError::NotConfigured)?;
let mut req = ctx.client.request(method, format!("{base}{path}")).bearer_auth(token).timeout(std::time::Duration::from_secs(20));
let mut req = ctx
.client
.request(method, format!("{base}{path}"))
.bearer_auth(token)
.timeout(std::time::Duration::from_secs(20));
// The directory's spans join this request's trace (#213).
if let Some(tp) = ipx_directory::otel::traceparent() {
req = req.header("traceparent", tp);
@@ -80,13 +93,19 @@ async fn call<T: DeserializeOwned>(ctx: &Ctx, method: reqwest::Method, path: &st
let resp = req.send().await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
let status = resp.status();
if !status.is_success() {
let msg = resp.json::<serde_json::Value>().await.ok().and_then(|v| v["error"].as_str().map(str::to_owned)).unwrap_or_default();
let msg = resp
.json::<serde_json::Value>()
.await
.ok()
.and_then(|v| v["error"].as_str().map(str::to_owned))
.unwrap_or_default();
return Err(DirError::Refused(status.as_u16(), msg));
}
// A removal or an unblock answers 204, with nothing to read: that reads as null, which is ().
let body = resp.bytes().await.map_err(|e| DirError::Unreachable(format!("{e:#}")))?;
tracing::Span::current().record("http.response.body.size", body.len());
serde_json::from_slice(if body.is_empty() { b"null" } else { &body }).map_err(|e| DirError::Unreachable(format!("its answer did not read: {e:#}")))
serde_json::from_slice(if body.is_empty() { b"null" } else { &body })
.map_err(|e| DirError::Unreachable(format!("its answer did not read: {e:#}")))
}
fn seg(s: &str) -> percent_encoding::PercentEncode<'_> {
@@ -211,7 +230,8 @@ pub async fn run(ctx: std::sync::Arc<Ctx>, mut stop: tokio::sync::watch::Receive
let key = serde_json::to_vec(&(target(&cfg), &entries)).unwrap_or_default();
let forced = ctx.directory_resend.swap(false, std::sync::atomic::Ordering::Relaxed);
if forced || resend(last.as_ref(), &key) {
let span = tracing::info_span!(target: "ipx::directory", "directory_share", feeds = entries.len());
let span =
tracing::info_span!(target: "ipx::directory", "directory_share", feeds = entries.len());
let got = tracing::Instrument::instrument(share(&ctx, &entries), span).await;
if got.is_ok() {
last = Some((key, std::time::Instant::now()));
@@ -225,7 +245,12 @@ pub async fn run(ctx: std::sync::Arc<Ctx>, mut stop: tokio::sync::watch::Receive
};
if let Some(got) = got {
match &got {
Ok(r) => tracing::info!(accepted = r.accepted, refused_private = r.refused_private, refused_blocked = r.refused_blocked, "shared with the directory"),
Ok(r) => tracing::info!(
accepted = r.accepted,
refused_private = r.refused_private,
refused_blocked = r.refused_blocked,
"shared with the directory"
),
Err(e) => tracing::warn!(error = %e, "could not share with the directory"),
}
let (result, error) = match got {

84
crates/ipx/src/download.rs
View File

@@ -85,16 +85,8 @@ pub fn filename_for(url: &str, content_disposition: Option<&str>) -> String {
}
let from_url = url::Url::parse(url)
.ok()
.and_then(|u| {
u.path_segments()
.and_then(|s| s.filter(|p| !p.is_empty()).next_back())
.map(str::to_owned)
})
.map(|seg| {
percent_encoding::percent_decode_str(&seg)
.decode_utf8_lossy()
.into_owned()
})
.and_then(|u| u.path_segments().and_then(|s| s.filter(|p| !p.is_empty()).next_back()).map(str::to_owned))
.map(|seg| percent_encoding::percent_decode_str(&seg).decode_utf8_lossy().into_owned())
.unwrap_or_default();
sanitize(&from_url)
}
@@ -105,11 +97,7 @@ fn disposition_filename(header: &str) -> Option<String> {
// RFC 5987 form wins when both are present.
if let Some(v) = part.strip_prefix("filename*=") {
let raw = v.rsplit('\'').next().unwrap_or(v);
return Some(
percent_encoding::percent_decode_str(raw)
.decode_utf8_lossy()
.into_owned(),
);
return Some(percent_encoding::percent_decode_str(raw).decode_utf8_lossy().into_owned());
}
}
for part in header.split(';') {
@@ -155,21 +143,15 @@ pub async fn download(
let name = filename_for(
resp.url().as_str(),
resp.headers()
.get(reqwest::header::CONTENT_DISPOSITION)
.and_then(|v| v.to_str().ok()),
resp.headers().get(reqwest::header::CONTENT_DISPOSITION).and_then(|v| v.to_str().ok()),
);
let total = resp.content_length();
let tmp_dir = cfg.general.download_dir.join(".ipx-incomplete");
tokio::fs::create_dir_all(&tmp_dir)
.await
.with_context(|| format!("creating {}", tmp_dir.display()))?;
tokio::fs::create_dir_all(&tmp_dir).await.with_context(|| format!("creating {}", tmp_dir.display()))?;
let tmp = tmp_dir.join(&name);
let mut file = tokio::fs::File::create(&tmp)
.await
.with_context(|| format!("creating {}", tmp.display()))?;
let mut file = tokio::fs::File::create(&tmp).await.with_context(|| format!("creating {}", tmp.display()))?;
let mut done: u64 = 0;
let mut stream = resp.bytes_stream();
let _active = crate::metrics::busy(&crate::metrics::STATS.downloads_active);
@@ -199,13 +181,9 @@ pub async fn download(
/// Moves a completed download into its folder. Same filesystem as the incomplete dir, so
/// this is a rename rather than the original's copy-then-unlink.
pub async fn place(d: &Downloaded, dir: &Path) -> Result<PathBuf> {
tokio::fs::create_dir_all(dir)
.await
.with_context(|| format!("creating {}", dir.display()))?;
tokio::fs::create_dir_all(dir).await.with_context(|| format!("creating {}", dir.display()))?;
let dest = unique_path(dir, &d.name);
tokio::fs::rename(&d.tmp, &dest)
.await
.with_context(|| format!("moving into {}", dest.display()))?;
tokio::fs::rename(&d.tmp, &dest).await.with_context(|| format!("moving into {}", dest.display()))?;
Ok(dest)
}
@@ -281,15 +259,8 @@ fn unique_path(dir: &Path, name: &str) -> PathBuf {
if !candidate.exists() {
return candidate;
}
let stem = Path::new(name)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or(name);
let ext = Path::new(name)
.extension()
.and_then(|e| e.to_str())
.map(|e| format!(".{e}"))
.unwrap_or_default();
let stem = Path::new(name).file_stem().and_then(|s| s.to_str()).unwrap_or(name);
let ext = Path::new(name).extension().and_then(|e| e.to_str()).map(|e| format!(".{e}")).unwrap_or_default();
for n in 2..1000 {
let candidate = dir.join(format!("{stem}-{n}{ext}"));
if !candidate.exists() {
@@ -308,12 +279,7 @@ pub fn folder_for(cfg: &Config, id: &str, feed_cfg: &FeedCfg, title: Option<&str
match cfg.general.organize {
Organize::Date => chrono::Local::now().format("%m-%d-%Y").to_string(),
Organize::Feed => {
let raw = feed_cfg
.folder
.as_deref()
.or(title)
.filter(|s| !s.trim().is_empty())
.unwrap_or(id);
let raw = feed_cfg.folder.as_deref().or(title).filter(|s| !s.trim().is_empty()).unwrap_or(id);
raw.split('/')
.map(str::trim)
.filter(|seg| !seg.is_empty() && *seg != "." && *seg != "..")
@@ -410,19 +376,13 @@ mod tests {
"ep1.mp3",
"query string is not part of the name"
);
assert_eq!(
filename_for("https://x.com/media/My%20Show%20Ep%201.mp3", None),
"My Show Ep 1.mp3"
);
assert_eq!(filename_for("https://x.com/media/My%20Show%20Ep%201.mp3", None), "My Show Ep 1.mp3");
assert_eq!(
filename_for("https://x.com/dl?id=9", Some("attachment; filename=\"real name.mp3\"")),
"real name.mp3",
"Content-Disposition overrides the URL"
);
assert_eq!(
filename_for("https://x.com/dl", Some("attachment; filename*=UTF-8''caf%C3%A9.mp3")),
"café.mp3"
);
assert_eq!(filename_for("https://x.com/dl", Some("attachment; filename*=UTF-8''caf%C3%A9.mp3")), "café.mp3");
assert_eq!(filename_for("https://x.com/", None), "download");
assert_eq!(
filename_for("https://x.com/a/../../etc/passwd", None),
@@ -436,9 +396,21 @@ mod tests {
let mut cfg = Config::default();
cfg.general.download_dir = "/tmp".into();
let mut f = crate::config::Feed {
url: "u".into(), folder: Some("Subscriptions/Some | Show".into()), group: None, media_types: None,
schedule: None, keywords: vec![], allow_explicit: false, auto_download: true,
max_new_per_check: None, username: None, password: None, password_env: None, category: None, listed: false, private: false,
url: "u".into(),
folder: Some("Subscriptions/Some | Show".into()),
group: None,
media_types: None,
schedule: None,
keywords: vec![],
allow_explicit: false,
auto_download: true,
max_new_per_check: None,
username: None,
password: None,
password_env: None,
category: None,
listed: false,
private: false,
};
assert_eq!(folder_for(&cfg, "id", &f, None), "Subscriptions/Some - Show");

139
crates/ipx/src/ipc.rs
View File

@@ -11,13 +11,31 @@ use tokio::sync::{broadcast, mpsc};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "ev", rename_all = "snake_case")]
pub enum Event {
FeedStart { feed: String },
FeedSkip { feed: String, reason: String },
FeedDone { feed: String, new: usize, downloaded: usize, failed: usize, torrents: usize },
FeedError { feed: String, msg: String },
FeedStart {
feed: String,
},
FeedSkip {
feed: String,
reason: String,
},
FeedDone {
feed: String,
new: usize,
downloaded: usize,
failed: usize,
torrents: usize,
},
FeedError {
feed: String,
msg: String,
},
/// The feed answered from a new address after redirects that all said it moved for good,
/// and the catalogue now has that address (#115).
FeedMoved { feed: String, from: String, to: String },
FeedMoved {
feed: String,
from: String,
to: String,
},
Progress {
feed: String,
/// Which enclosure this is about. Without it a UI cannot tell one download's
@@ -29,14 +47,38 @@ pub enum Event {
#[serde(skip_serializing_if = "Option::is_none")]
total: Option<u64>,
},
DownloadDone { feed: String, enclosure: i64, url: String, path: String, bytes: u64 },
DownloadError { feed: String, enclosure: i64, url: String, msg: String },
TorrentDeferred { feed: String, url: String },
DownloadDone {
feed: String,
enclosure: i64,
url: String,
path: String,
bytes: u64,
},
DownloadError {
feed: String,
enclosure: i64,
url: String,
msg: String,
},
TorrentDeferred {
feed: String,
url: String,
},
/// DeadAir finished a file's copy (#171), so a page showing it as in progress can stop.
DeadairDone { feed: String, enclosure: i64, saved_s: i64 },
DeadairDone {
feed: String,
enclosure: i64,
saved_s: i64,
},
/// Jev found items for someone's Signals (#258), so their page fetches the counts again.
SignalHits { user: i64, hits: usize },
Reaped { path: String, bytes: u64 },
SignalHits {
user: i64,
hits: usize,
},
Reaped {
path: String,
bytes: u64,
},
/// Terminal: a client that asked for work stops reading here.
ScanDone {
feeds: usize,
@@ -47,9 +89,18 @@ pub enum Event {
#[serde(default)]
forced: bool,
},
ReapDone { files: usize, bytes: u64 },
Status { feeds: usize, pending: i64, downloaded: i64 },
Error { msg: String },
ReapDone {
files: usize,
bytes: u64,
},
Status {
feeds: usize,
pending: i64,
downloaded: i64,
},
Error {
msg: String,
},
}
impl Event {
@@ -80,10 +131,9 @@ impl Event {
Event::Reaped { path, bytes } => {
format!("deleted {path} ({:.1} MB)", *bytes as f64 / 1_048_576.0)
}
Event::ReapDone { files, bytes } => format!(
"deleted {files} old file(s), {:.1} MB",
*bytes as f64 / 1_048_576.0
),
Event::ReapDone { files, bytes } => {
format!("deleted {files} old file(s), {:.1} MB", *bytes as f64 / 1_048_576.0)
}
Event::Status { feeds, pending, downloaded } => {
format!("{feeds} feeds, {pending} queued to download, {downloaded} downloaded")
}
@@ -91,7 +141,9 @@ impl Event {
// Noise in a terminal; a UI still gets them on the socket.
Event::FeedStart { feed } => format!("{feed}: checking"),
Event::TorrentDeferred { feed, .. } => format!("{feed}: torrent deferred"),
Event::DeadairDone { feed, saved_s, .. } => format!("{feed}: DeadAir found the silences, {saved_s} s shorter"),
Event::DeadairDone { feed, saved_s, .. } => {
format!("{feed}: DeadAir found the silences, {saved_s} s shorter")
}
Event::SignalHits { hits, .. } => format!("{hits} item(s) picked out for Signals"),
Event::ScanDone { feeds, .. } => format!("scan complete, {feeds} feed(s)"),
})
@@ -165,7 +217,9 @@ impl Emitter {
// An error here only means nobody is listening yet.
let _ = tx.send(e.clone());
}
if self.print && let Some(line) = e.human() {
if self.print
&& let Some(line) = e.human()
{
println!("{line}");
}
}
@@ -262,9 +316,8 @@ pub async fn daemon_is_live(path: &Path) -> bool {
/// the meantime. The answer goes to the client that asked and no one else: broadcast, it ended
/// any `ipx fetch` that was watching a scan, since `status` is a terminal event.
/// A future, since reading the counts is a database query.
pub type StatusFn = std::sync::Arc<
dyn Fn(Command) -> std::pin::Pin<Box<dyn std::future::Future<Output = Event> + Send>> + Send + Sync,
>;
pub type StatusFn =
std::sync::Arc<dyn Fn(Command) -> std::pin::Pin<Box<dyn std::future::Future<Output = Event> + Send>> + Send + Sync>;
/// Accepts connections, feeding commands to `cmds` and events from `events` back out.
pub async fn serve(
@@ -276,14 +329,12 @@ pub async fn serve(
// A socket file left by a crashed daemon would block the bind; a live one was already
// rejected by the caller's daemon_is_live() check.
if path.exists() {
std::fs::remove_file(&path)
.with_context(|| format!("removing stale socket {}", path.display()))?;
std::fs::remove_file(&path).with_context(|| format!("removing stale socket {}", path.display()))?;
}
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
let listener = UnixListener::bind(&path)
.with_context(|| format!("binding {}", path.display()))?;
let listener = UnixListener::bind(&path).with_context(|| format!("binding {}", path.display()))?;
tracing::info!(socket = %path.display(), "listening");
loop {
@@ -370,7 +421,9 @@ pub async fn reload(path: &Path) -> Result<()> {
}
anyhow::bail!("the daemon closed the connection without answering")
};
tokio::time::timeout(std::time::Duration::from_secs(10), answer).await.context("the daemon did not answer in 10s")?
tokio::time::timeout(std::time::Duration::from_secs(10), answer)
.await
.context("the daemon did not answer in 10s")?
}
/// Sends one command to a running daemon and prints the events it produces.
@@ -428,7 +481,11 @@ mod tests {
url: format!("https://cdn/ep.mp3?{key}"),
msg: format!("connecting: error sending request for url (https://cdn/ep.mp3?{key})"),
});
out.emit(Event::FeedMoved { feed: "f".into(), from: format!("https://x/rss?{key}"), to: format!("https://y/rss?{key}") });
out.emit(Event::FeedMoved {
feed: "f".into(),
from: format!("https://x/rss?{key}"),
to: format!("https://y/rss?{key}"),
});
// What every open page is sent, whoever's feed it is (#238).
for _ in 0..3 {
let sent = serde_json::to_string(&rx.try_recv().unwrap()).unwrap();
@@ -441,7 +498,10 @@ mod tests {
fn a_feed_that_moved_says_so_on_the_wire_and_in_words() {
let e = Event::FeedMoved { feed: "x".into(), from: "http://a/f".into(), to: "https://a/f".into() };
let wire = serde_json::to_value(&e).unwrap();
assert_eq!((wire["ev"].as_str(), wire["from"].as_str(), wire["to"].as_str()), (Some("feed_moved"), Some("http://a/f"), Some("https://a/f")));
assert_eq!(
(wire["ev"].as_str(), wire["from"].as_str(), wire["to"].as_str()),
(Some("feed_moved"), Some("http://a/f"), Some("https://a/f"))
);
assert_eq!(e.human().unwrap(), "x: moved for good from http://a/f to https://a/f; following it");
}
@@ -480,14 +540,8 @@ mod tests {
assert!(json.contains(r#""enclosure":42"#), "a UI needs this to target one row");
// total is omitted rather than null when the server sent no length.
let ev = Event::Progress {
feed: "a".into(),
enclosure: 1,
url: "u".into(),
file: "f".into(),
done: 1,
total: None,
};
let ev =
Event::Progress { feed: "a".into(), enclosure: 1, url: "u".into(), file: "f".into(), done: 1, total: None };
assert!(!serde_json::to_string(&ev).unwrap().contains("total"));
}
@@ -495,14 +549,7 @@ mod tests {
fn only_completion_events_end_a_client_session() {
assert!(Event::ScanDone { feeds: 1, only: None, forced: false }.is_terminal());
assert!(Event::ReapDone { files: 0, bytes: 0 }.is_terminal());
assert!(!Event::FeedDone {
feed: "a".into(),
new: 0,
downloaded: 0,
failed: 0,
torrents: 0
}
.is_terminal());
assert!(!Event::FeedDone { feed: "a".into(), new: 0, downloaded: 0, failed: 0, torrents: 0 }.is_terminal());
}
#[tokio::test]

17
crates/ipx/src/logbuf.rs
View File

@@ -27,9 +27,8 @@ struct Ring {
next_seq: u64,
}
static BUF: LazyLock<Mutex<Ring>> = LazyLock::new(|| {
Mutex::new(Ring { lines: VecDeque::with_capacity(CAPACITY), next_seq: 1 })
});
static BUF: LazyLock<Mutex<Ring>> =
LazyLock::new(|| Mutex::new(Ring { lines: VecDeque::with_capacity(CAPACITY), next_seq: 1 }));
pub fn push(level: &str, target: &str, msg: String) {
let mut ring = match BUF.lock() {
@@ -57,12 +56,7 @@ pub fn since(after: u64, limit: usize) -> (Vec<LogLine>, u64) {
Err(p) => p.into_inner(),
};
let latest = ring.next_seq.saturating_sub(1);
let mut out: Vec<LogLine> = ring
.lines
.iter()
.filter(|l| l.seq > after)
.cloned()
.collect();
let mut out: Vec<LogLine> = ring.lines.iter().filter(|l| l.seq > after).cloned().collect();
// On a first load (after = 0) the tail is what matters, not the head.
if out.len() > limit {
out.drain(..out.len() - limit);
@@ -130,10 +124,7 @@ mod tests {
let (all, latest) = since(0, CAPACITY * 2);
assert_eq!(all.len(), CAPACITY, "bounded");
assert!(latest >= (CAPACITY + 50) as u64);
assert!(
all.first().unwrap().seq < all.last().unwrap().seq,
"oldest first"
);
assert!(all.first().unwrap().seq < all.last().unwrap().seq, "oldest first");
// "everything after the last one I saw" must return nothing new.
let (none, _) = since(latest, 100);

414
crates/ipx/src/main.rs
View File

@@ -5,8 +5,8 @@ mod config;
mod db;
mod deadair;
mod directory;
mod entity;
mod download;
mod entity;
// The crate root's import, so `crate::feed` reaches it from every module as before.
use ipx_feed as feed;
mod ipc;
@@ -20,9 +20,9 @@ mod web;
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use ipc::{Command as Cmd, Emitter, Event};
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
use std::future::Future;
use tokio::sync::{broadcast, mpsc, watch};
#[derive(Parser)]
@@ -200,7 +200,14 @@ const RETRY_LOOKUP_SECS: i64 = 600;
impl Ctx {
/// When a feed is next due, taking a lookup retry into account: not before it.
fn due(&self, cfg: &config::Config, feed: &config::Feed, id: &str, state: &db::HttpState, subscribed: bool) -> Option<i64> {
fn due(
&self,
cfg: &config::Config,
feed: &config::Feed,
id: &str,
state: &db::HttpState,
subscribed: bool,
) -> Option<i64> {
let at = due_at(cfg, feed, state, subscribed);
match self.retry.lock().unwrap().get(id) {
Some(&retry) => Some(at.unwrap_or(0).max(retry)),
@@ -250,9 +257,7 @@ impl Ctx {
async fn torrents(&self) -> Result<&torrent::Torrents> {
let cfg = self.cfg();
self.torrents
.get_or_try_init(|| torrent::Torrents::new(&cfg))
.await
self.torrents.get_or_try_init(|| torrent::Torrents::new(&cfg)).await
}
}
@@ -267,20 +272,19 @@ async fn main() -> Result<()> {
};
// Everything goes to stderr as before, and is mirrored into a ring the UI can read.
{
use tracing_subscriber::Layer;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::Layer;
// Two filters, deliberately different. stderr follows IPX_LOG; the in-app buffer
// keeps debug as well, so the log view can show protocol traffic and routine
// skips that would be noise on a terminal. IPX_UI_LOG overrides it.
let stderr_filter = || {
tracing_subscriber::EnvFilter::try_from_env("IPX_LOG").unwrap_or_else(|_| "ipx=info".into())
};
let stderr_filter =
|| tracing_subscriber::EnvFilter::try_from_env("IPX_LOG").unwrap_or_else(|_| "ipx=info".into());
// One JSON object a line for Loki (#91), with each event's fields as its own; text
// otherwise, for someone reading a terminal.
let json = std::env::var("IPX_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
let ui_filter = tracing_subscriber::EnvFilter::try_from_env("IPX_UI_LOG")
.unwrap_or_else(|_| "ipx=debug".into());
let ui_filter =
tracing_subscriber::EnvFilter::try_from_env("IPX_UI_LOG").unwrap_or_else(|_| "ipx=debug".into());
tracing_subscriber::registry()
.with((!json).then(|| {
tracing_subscriber::fmt::layer()
@@ -294,7 +298,11 @@ async fn main() -> Result<()> {
tracing_subscriber::fmt::layer()
.fmt_fields(tracing_subscriber::fmt::format::JsonFields::new())
.event_format(WithTrace(
tracing_subscriber::fmt::format().json().flatten_event(true).with_current_span(true).with_span_list(false),
tracing_subscriber::fmt::format()
.json()
.flatten_event(true)
.with_current_span(true)
.with_span_list(false),
))
.with_writer(std::io::stderr)
.with_filter(stderr_filter())
@@ -315,9 +323,7 @@ async fn main() -> Result<()> {
// A daemon owns the state; don't have two processes downloading the same thing.
let wire_cmd = match &cli.command {
Command::Fetch { feed, force } => {
Some(Cmd::Fetch { feed: feed.clone(), force: *force, feeds: vec![] })
}
Command::Fetch { feed, force } => Some(Cmd::Fetch { feed: feed.clone(), force: *force, feeds: vec![] }),
Command::Reap { dry_run } => Some(Cmd::Reap { dry_run: *dry_run }),
Command::Status => Some(Cmd::Status),
Command::List
@@ -459,11 +465,7 @@ async fn user_cmd(ctx: &Arc<Ctx>, cmd: UserCmd) -> Result<()> {
if ctx.db.user_by_name(&name).await?.is_some() {
anyhow::bail!("{name} already exists");
}
let hash = if no_password {
None
} else {
Some(crate::auth::hash_password(&read_password()?)?)
};
let hash = if no_password { None } else { Some(crate::auth::hash_password(&read_password()?)?) };
// The first account runs the place; there is nobody else to grant it.
let first = ctx.db.users().await?.is_empty();
ctx.db.create_user(&name, hash.as_deref(), admin || first).await?;
@@ -496,10 +498,7 @@ async fn user_cmd(ctx: &Arc<Ctx>, cmd: UserCmd) -> Result<()> {
}
UserCmd::Passwd { name } => {
let name = name.trim().to_ascii_lowercase();
let user = ctx
.db
.user_by_name(&name).await?
.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
let user = ctx.db.user_by_name(&name).await?.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
ctx.db.set_password(user.id, &crate::auth::hash_password(&read_password()?)?).await?;
println!("password changed for {name}");
Ok(())
@@ -509,10 +508,7 @@ async fn user_cmd(ctx: &Arc<Ctx>, cmd: UserCmd) -> Result<()> {
// The same rules as a name the proxy vouches for, or the proxy would never find it.
let new_name = crate::auth::name_from_header(&new_name)
.ok_or_else(|| anyhow::anyhow!("not a usable name: no commas, semicolons or line breaks"))?;
let user = ctx
.db
.user_by_name(&name).await?
.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
let user = ctx.db.user_by_name(&name).await?.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
if ctx.db.user_by_name(&new_name).await?.is_some() {
anyhow::bail!("{new_name} already exists");
}
@@ -522,10 +518,7 @@ async fn user_cmd(ctx: &Arc<Ctx>, cmd: UserCmd) -> Result<()> {
}
UserCmd::Rm { name } => {
let name = name.trim().to_ascii_lowercase();
let user = ctx
.db
.user_by_name(&name).await?
.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
let user = ctx.db.user_by_name(&name).await?.ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?;
remove_user(ctx, user.id).await?;
println!("removed {name}");
Ok(())
@@ -654,10 +647,7 @@ async fn daemon(
let server = tokio::spawn(ipc::serve(socket.clone(), events.clone(), tx_cmd, answer));
// One command at a time: the queue is what keeps two scans from overlapping.
tracing::info!(
feeds = subscriptions(&ctx).await.map(|s| s.len()).unwrap_or(0),
"daemon started"
);
tracing::info!(feeds = subscriptions(&ctx).await.map(|s| s.len()).unwrap_or(0), "daemon started");
// A signal has to be able to interrupt work in progress, not just the wait between
// jobs. Racing `shutdown()` in the outer select only cancels branch selection: once
@@ -689,11 +679,7 @@ async fn daemon(
}
// Runs one job, abandoning it if a signal arrives. Returns false to end the loop.
async fn until_stopped(
ctx: &Ctx,
rx: &watch::Receiver<bool>,
job: impl Future<Output = Result<()>>,
) -> bool {
async fn until_stopped(ctx: &Ctx, rx: &watch::Receiver<bool>, job: impl Future<Output = Result<()>>) -> bool {
let mut stop = rx.clone();
tokio::select! {
_ = stop.changed() => {
@@ -783,10 +769,7 @@ async fn start_web(
config_path.display()
);
} else {
tracing::info!(
"web ui at http://{bind}/ (the sign-in token is [web] token in {})",
config_path.display()
);
tracing::info!("web ui at http://{bind}/ (the sign-in token is [web] token in {})", config_path.display());
}
// Bound beyond localhost, as a container has to be for its port to be published. Worth a
@@ -825,13 +808,7 @@ async fn start_web(
let at = metrics::spawn(&cfg.web.metrics_bind, sources)?;
tracing::info!(%at, "serving metrics for Prometheus at /metrics");
}
let state = web::WebState {
ctx: ctx.clone(),
cmds: cmds.clone(),
events: events.clone(),
access,
playing,
};
let state = web::WebState { ctx: ctx.clone(), cmds: cmds.clone(), events: events.clone(), access, playing };
Ok(Some(tokio::spawn(async move {
if let Err(e) = web::serve(state, &bind).await {
tracing::error!(error = %format!("{e:#}"), "web ui stopped");
@@ -938,10 +915,9 @@ async fn add_one(
Ok(feed::Fetched::Body { bytes, .. }) if feed::is_opml(&bytes) => {
feed::opml_title(&bytes).unwrap_or_else(|| url_stem(url))
}
Ok(feed::Fetched::Body { bytes, .. }) => feed::parse(&bytes)
.ok()
.and_then(|f| f.title)
.unwrap_or_else(|| url_stem(url)),
Ok(feed::Fetched::Body { bytes, .. }) => {
feed::parse(&bytes).ok().and_then(|f| f.title).unwrap_or_else(|| url_stem(url))
}
_ => {
tracing::warn!(url, "could not read the feed; naming it from its URL");
url_stem(url)
@@ -950,10 +926,8 @@ async fn add_one(
// Slugs must be unique across derived feeds too, or a new feed can collide with one
// an OPML already introduced.
let mut taken: std::collections::BTreeMap<String, config::Feed> = subscriptions(ctx).await?
.into_iter()
.map(|s| (s.id, s.cfg))
.collect();
let mut taken: std::collections::BTreeMap<String, config::Feed> =
subscriptions(ctx).await?.into_iter().map(|s| (s.id, s.cfg)).collect();
// A removed feed keeps its rows, so its id is only free again for the same feed: re-adding
// it gets its history back, and a different feed does not inherit someone else's.
for (id, other) in ctx.db.feed_urls().await? {
@@ -968,10 +942,7 @@ async fn add_one(
/// Host plus last path segment, for naming a feed we could not read.
fn url_stem(url: &str) -> String {
url::Url::parse(url)
.ok()
.and_then(|u| u.host_str().map(str::to_owned))
.unwrap_or_else(|| url.to_owned())
url::Url::parse(url).ok().and_then(|u| u.host_str().map(str::to_owned)).unwrap_or_else(|| url.to_owned())
}
async fn rm(ctx: &Ctx, feed: &str) -> Result<()> {
@@ -991,17 +962,16 @@ async fn rm(ctx: &Ctx, feed: &str) -> Result<()> {
}
async fn import(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
let text = std::fs::read_to_string(file)
.with_context(|| format!("reading {}", file.display()))?;
let text = std::fs::read_to_string(file).with_context(|| format!("reading {}", file.display()))?;
// The CLI speaks for the operator, as the shared web token does.
let admin = ctx
.db
.users().await?
.users()
.await?
.into_iter()
.find(|u| u.is_admin)
.ok_or_else(|| anyhow::anyhow!("no admin account to subscribe: ipx user add <name> --admin"))?;
let doc = opml::OPML::from_str(&text)
.map_err(|e| anyhow::anyhow!("{} is not OPML: {e}", file.display()))?;
let doc = opml::OPML::from_str(&text).map_err(|e| anyhow::anyhow!("{} is not OPML: {e}", file.display()))?;
let (added, had) = subscribe_opml(ctx, &doc, admin.id).await?;
println!("subscribed {} to {added} feed(s); {had} already there", admin.name);
Ok(())
@@ -1017,11 +987,7 @@ async fn import(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
///
/// The caller parses the document, so each refuses a file that is not OPML in its own terms,
/// before anything is touched: a 400 from the web, a message from the CLI.
pub async fn subscribe_opml(
ctx: &Ctx,
doc: &opml::OPML,
user_id: i64,
) -> Result<(usize, usize)> {
pub async fn subscribe_opml(ctx: &Ctx, doc: &opml::OPML, user_id: i64) -> Result<(usize, usize)> {
let mut found = vec![];
feed::collect_outlines(&doc.body.outlines, &mut found);
@@ -1108,7 +1074,10 @@ async fn assemble_config(db: &db::Db, mut cfg: config::Config, path: &std::path:
}
cfg.save_bootstrap(path)?;
}
tracing::info!(feeds = cfg.feeds.len(), "moved the feeds and server settings from config.toml into the database");
tracing::info!(
feeds = cfg.feeds.len(),
"moved the feeds and server settings from config.toml into the database"
);
return Ok(cfg);
}
// Another ipx imported between our look and our insert; the next pass reads its copy.
@@ -1118,17 +1087,9 @@ async fn assemble_config(db: &db::Db, mut cfg: config::Config, path: &std::path:
async fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
let mut doc = opml::OPML::default();
doc.head = Some(opml::Head {
title: Some("ipx subscriptions".into()),
..Default::default()
});
doc.head = Some(opml::Head { title: Some("ipx subscriptions".into()), ..Default::default() });
for (id, feed) in &ctx.cfg().feeds {
let title = ctx
.db
.feed_summary(id).await
.ok()
.and_then(|s| s.title)
.unwrap_or_else(|| id.clone());
let title = ctx.db.feed_summary(id).await.ok().and_then(|s| s.title).unwrap_or_else(|| id.clone());
doc.add_feed(&title, &feed.url);
}
let xml = doc.to_string().map_err(|e| anyhow::anyhow!("writing OPML: {e}"))?;
@@ -1184,7 +1145,10 @@ async fn clean_directory(ctx: &Ctx) -> Result<()> {
for id in &stale {
retire_group(ctx, id).await?;
ctx.db.forget_feed(id).await?;
tracing::info!(feed = id, "removed a feed nobody subscribes to that is dead or has published nothing in a year");
tracing::info!(
feed = id,
"removed a feed nobody subscribes to that is dead or has published nothing in a year"
);
}
Ok(())
}
@@ -1201,22 +1165,17 @@ async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
span.record("entries", r.entries_pruned);
if !dry_run {
tracing::Instrument::instrument(clean_directory(ctx), db::step("clean_directory")).await?;
let gone = db::step("trim_artwork").in_scope(|| art::trim(&art::dir(), ctx.cfg().general.art_cache_mb * 1_048_576));
let gone =
db::step("trim_artwork").in_scope(|| art::trim(&art::dir(), ctx.cfg().general.art_cache_mb * 1_048_576));
if gone > 0 {
tracing::debug!(gone, "trimmed the artwork kept on disk");
}
}
for c in r.aged_out.iter().chain(r.over_quota.iter()) {
ctx.out.emit(Event::Reaped {
path: c.path.clone(),
bytes: c.bytes.max(0) as u64,
});
ctx.out.emit(Event::Reaped { path: c.path.clone(), bytes: c.bytes.max(0) as u64 });
}
if standalone {
ctx.out.emit(Event::ReapDone {
files: r.aged_out.len() + r.over_quota.len(),
bytes: r.bytes_freed,
});
ctx.out.emit(Event::ReapDone { files: r.aged_out.len() + r.over_quota.len(), bytes: r.bytes_freed });
}
Ok(())
}
@@ -1235,9 +1194,8 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
let subscribed = ctx.db.subscriber_counts().await?;
let mut scanned = 0;
let mut fresh: Vec<String> = vec![];
let in_scope = |s: &Sub| {
scope.is_empty() || scope.contains(&s.id) || s.cfg.group.as_ref().is_some_and(|g| scope.contains(g))
};
let in_scope =
|s: &Sub| scope.is_empty() || scope.contains(&s.id) || s.cfg.group.as_ref().is_some_and(|g| scope.contains(g));
let mut states = ctx.db.http_states().await?;
let mut due = vec![];
for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id) && in_scope(s)) {
@@ -1281,7 +1239,9 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
return None;
}
let _fetching = metrics::busy(&metrics::STATS.feeds_fetching);
let got = feed::fetch(&client, &feed_cfg.url, feed_cfg.auth(), etag.as_deref(), modified.as_deref()).await;
let got =
feed::fetch(&client, &feed_cfg.url, feed_cfg.auth(), etag.as_deref(), modified.as_deref())
.await;
// A forced scan looks for the site's icon of each feed naming no artwork: here,
// a few feeds at a time, not each in its turn, 22 s of a 62 s scan (#228).
let icon = match &got {
@@ -1314,14 +1274,10 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool, scope: &[String]
failed: s.failed,
torrents: s.torrents,
}),
Ok(Outcome::NotModified) => ctx.out.emit(Event::FeedSkip {
feed: id.clone(),
reason: "not modified".into(),
}),
Ok(Outcome::Empty) => ctx.out.emit(Event::FeedSkip {
feed: id.clone(),
reason: "nothing yet".into(),
}),
Ok(Outcome::NotModified) => {
ctx.out.emit(Event::FeedSkip { feed: id.clone(), reason: "not modified".into() })
}
Ok(Outcome::Empty) => ctx.out.emit(Event::FeedSkip { feed: id.clone(), reason: "nothing yet".into() }),
Ok(Outcome::Opml { added, removed, kept, total }) => {
ctx.out.emit(Event::FeedSkip {
feed: id.clone(),
@@ -1390,11 +1346,8 @@ pub struct Sub {
/// entry -- there is nothing to store but its URL and where it came from.
pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> {
let cfg = ctx.cfg();
let mut out: Vec<Sub> = cfg
.feeds
.iter()
.map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false })
.collect();
let mut out: Vec<Sub> =
cfg.feeds.iter().map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false }).collect();
// A group's title, once a group: asked for each feed in it, three queries a feed, it was about
// 96 for The Glass Cannon Network's 32 shows on every feed list, live update and scan (#159).
@@ -1453,7 +1406,9 @@ pub async fn leave(ctx: &Ctx, user_id: i64, id: &str) -> Result<()> {
ctx.db.unsubscribe(user_id, id).await?;
// What they kept stays on their list, a feed of its own once nobody reads the list (#280).
let kept = ctx.db.kept_feeds(user_id).await?;
for child in subscriptions(ctx).await?.iter().filter(|s| s.cfg.group.as_deref() == Some(id) && !kept.contains(&s.id)) {
for child in
subscriptions(ctx).await?.iter().filter(|s| s.cfg.group.as_deref() == Some(id) && !kept.contains(&s.id))
{
ctx.db.unsubscribe(user_id, &child.id).await?;
}
if ctx.db.subscriber_counts().await?.contains_key(id) {
@@ -1564,11 +1519,8 @@ async fn promote_kept(ctx: &Ctx, m: &db::Managed) -> Result<bool> {
async fn retire_stranded(ctx: &Ctx) -> Result<usize> {
let cfg = ctx.cfg();
let before = ctx.db.managed_feeds().await?;
let stranded: std::collections::BTreeSet<&str> = before
.iter()
.map(|m| m.group_id.as_str())
.filter(|g| !cfg.feeds.contains_key(*g))
.collect();
let stranded: std::collections::BTreeSet<&str> =
before.iter().map(|m| m.group_id.as_str()).filter(|g| !cfg.feeds.contains_key(*g)).collect();
for group in stranded {
retire_group(ctx, group).await?;
}
@@ -1636,7 +1588,10 @@ pub fn due_after(
let usual = match explicit {
Some(explicit) => explicit * 60,
// A schedule someone set is theirs; otherwise a feed long the same is checked less.
None => quiet(ttl_mins.unwrap_or(0).max(cfg.general.interval()) * 60, same_since.map(|s| (last_checked - s).max(0) as u64)),
None => quiet(
ttl_mins.unwrap_or(0).max(cfg.general.interval()) * 60,
same_since.map(|s| (last_checked - s).max(0) as u64),
),
};
let failing_for = error_since.map(|since| (last_checked - since).max(0) as u64);
backoff(usual, failing_for)
@@ -1682,7 +1637,12 @@ enum Outcome {
Empty,
Feed(Scan),
/// The URL is a list of feeds rather than a feed: an OPML, or a Patreon creator's shows.
Opml { added: Vec<String>, removed: usize, kept: usize, total: usize },
Opml {
added: Vec<String>,
removed: usize,
kept: usize,
total: usize,
},
}
/// A feed that answered from a new address after permanent redirects moves there in the
@@ -1749,7 +1709,16 @@ async fn scan_one(
let (mut fetched, moved) = match prefetched {
Some(got) => got?,
None => feed::fetch(&ctx.feed_client, &feed_cfg.url, feed_cfg.auth(), state.etag.as_deref(), state.last_modified.as_deref()).await?,
None => {
feed::fetch(
&ctx.feed_client,
&feed_cfg.url,
feed_cfg.auth(),
state.etag.as_deref(),
state.last_modified.as_deref(),
)
.await?
}
};
if let Some(to) = moved {
follow_move(ctx, id, &to).await?;
@@ -1859,17 +1828,19 @@ async fn scan_one(
ctx.db.secure_images(id, host).await?;
}
}
ctx.db.record_feed(
id,
&feed_cfg.url,
parsed.title.as_deref(),
etag.as_deref(),
last_modified.as_deref(),
parsed.ttl_mins,
parsed.image.as_deref(),
parsed.category.as_deref(),
parsed.description.as_deref(),
).await?;
ctx.db
.record_feed(
id,
&feed_cfg.url,
parsed.title.as_deref(),
etag.as_deref(),
last_modified.as_deref(),
parsed.ttl_mins,
parsed.image.as_deref(),
parsed.category.as_deref(),
parsed.description.as_deref(),
)
.await?;
if state.body_hash.as_deref() != Some(hash.as_str()) {
ctx.db.set_body(id, &hash).await?;
}
@@ -1914,37 +1885,42 @@ async fn scan_one(
let mut scan = Scan::default();
// Its own span: the time a feed spends after its fetch was untraced (#96).
let store = tracing::info_span!("store", items = parsed.entries.len());
tracing::Instrument::instrument(async {
for entry in &parsed.entries {
// What is not stored yet is inserted, and what the feed has changed since is written
// again (#141); the rest is left alone. A correction keeps the item's read state.
if (!known_items.contains(&entry.guid) || changed.contains(&entry.guid))
&& ctx.db.record_entry(id, entry).await?
{
scan.new_entries += 1;
}
for enc in &entry.enclosures {
// A URL not among this feed's files may still be another feed's: the insert says.
let was = if !known_files.contains(&enc.url) && ctx.db.record_enclosure(id, &entry.guid, enc).await? {
None
} else if let Some(reason) = skipped.get(&enc.url) {
Some(reason.as_str())
} else {
continue; // Settled: queued, downloaded, reaped, or another feed's file.
};
let now = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc);
if now != was {
match now {
Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason)).await?,
None => ctx.db.mark_enclosure(&enc.url, "pending", None).await?,
tracing::Instrument::instrument(
async {
for entry in &parsed.entries {
// What is not stored yet is inserted, and what the feed has changed since is written
// again (#141); the rest is left alone. A correction keeps the item's read state.
if (!known_items.contains(&entry.guid) || changed.contains(&entry.guid))
&& ctx.db.record_entry(id, entry).await?
{
scan.new_entries += 1;
}
for enc in &entry.enclosures {
// A URL not among this feed's files may still be another feed's: the insert says.
let was =
if !known_files.contains(&enc.url) && ctx.db.record_enclosure(id, &entry.guid, enc).await? {
None
} else if let Some(reason) = skipped.get(&enc.url) {
Some(reason.as_str())
} else {
continue; // Settled: queued, downloaded, reaped, or another feed's file.
};
let now = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc);
if now != was {
match now {
Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason)).await?,
None => ctx.db.mark_enclosure(&enc.url, "pending", None).await?,
}
}
}
}
}
anyhow::Ok(())
}, store).await?;
anyhow::Ok(())
},
store,
)
.await?;
if scan.new_entries > 0 {
ctx.db.rehide(id).await?; // what is new may hold someone's blocked words
ctx.db.rehide(id).await?; // what is new may hold someone's blocked words
ctx.signals.notify_one();
}
// A new item takes a place among the newest and the oldest of them leaves the queue; a file a
@@ -1961,10 +1937,7 @@ async fn scan_one(
if download::looks_like_torrent(&item.url, item.mime.as_deref()) {
if !ctx.cfg().torrent.enabled {
ctx.db.mark_enclosure(&item.url, "skipped", Some("torrents disabled")).await?;
ctx.out.emit(Event::TorrentDeferred {
feed: id.to_string(),
url: item.url.clone(),
});
ctx.out.emit(Event::TorrentDeferred { feed: id.to_string(), url: item.url.clone() });
scan.torrents += 1;
continue;
}
@@ -2031,12 +2004,7 @@ async fn scan_one(
Ok(Outcome::Feed(scan))
}
async fn sync_opml(
ctx: &Arc<Ctx>,
parent_id: &str,
parent: &config::Feed,
bytes: &[u8],
) -> Result<Outcome> {
async fn sync_opml(ctx: &Arc<Ctx>, parent_id: &str, parent: &config::Feed, bytes: &[u8]) -> Result<Outcome> {
let listed = feed::parse_opml(bytes)?;
if let Some(title) = feed::opml_title(bytes) {
ctx.db.set_title(parent_id, &title).await?;
@@ -2090,7 +2058,8 @@ async fn sync_group(
// subscription means. Their own feeds are untouched.
for id in ctx
.db
.managed_feeds().await?
.managed_feeds()
.await?
.iter()
.filter(|m| m.group_id == parent_id)
.map(|m| m.id.clone())
@@ -2144,10 +2113,7 @@ fn reject(
}
// Blog feeds put the article's header image in an <enclosure>; without this a text
// feed reads as a podcast full of episodes and fills the disk with artwork.
let wanted = feed_cfg
.media_types
.as_deref()
.unwrap_or(&cfg.general.media_types);
let wanted = feed_cfg.media_types.as_deref().unwrap_or(&cfg.general.media_types);
if !config::wanted_media(enc.mime.as_deref(), wanted) {
return Some("not audio or video");
}
@@ -2155,12 +2121,8 @@ fn reject(
return Some("explicit");
}
let categories = entry.categories.join(" ");
let haystacks = [
url,
entry.title.as_deref().unwrap_or(""),
entry.description.as_deref().unwrap_or(""),
categories.as_str(),
];
let haystacks =
[url, entry.title.as_deref().unwrap_or(""), entry.description.as_deref().unwrap_or(""), categories.as_str()];
let text = [entry.title.as_deref().unwrap_or(""), entry.description.as_deref().unwrap_or("")];
if !policy.wanted(&haystacks, &text) {
// Worded for whichever filter could have let it through, the keywords if none blocked it.
@@ -2194,9 +2156,9 @@ impl Policy {
/// One file serves everyone subscribed, so an item is wanted if anyone wants it. Keywords
/// look at `haystacks`, blocked words at `text`, the item's title and body only.
fn wanted(&self, haystacks: &[&str], text: &[&str]) -> bool {
self.wants.iter().any(|w| {
download::matches_keywords(&w.keywords, haystacks) && !download::blocked(&w.blocked, text)
})
self.wants
.iter()
.any(|w| download::matches_keywords(&w.keywords, haystacks) && !download::blocked(&w.blocked, text))
}
}
@@ -2225,21 +2187,15 @@ fn merge_policy(subs: &[db::Sub], feed_cfg: &config::Feed, global: usize) -> Pol
};
}
let mut policy = Policy {
auto_download: false,
allow_explicit: false,
wants: vec![],
budget: 0,
};
let mut policy = Policy { auto_download: false, allow_explicit: false, wants: vec![], budget: 0 };
for sub in subs {
if !sub.auto_download.unwrap_or(feed_cfg.auto_download) {
continue; // Not fetching for this person, so their wants add nothing.
}
policy.auto_download = true;
policy.allow_explicit |= sub.allow_explicit.unwrap_or(feed_cfg.allow_explicit);
policy.budget = policy
.budget
.max(cap(sub.max_new_per_check.map(|n| n as usize).or(feed_cfg.max_new_per_check)));
policy.budget =
policy.budget.max(cap(sub.max_new_per_check.map(|n| n as usize).or(feed_cfg.max_new_per_check)));
policy.wants.push(Want {
keywords: sub.keywords.clone().unwrap_or_else(|| feed_cfg.keywords.clone()),
blocked: sub.blocked.clone(),
@@ -2346,10 +2302,7 @@ fn spawn_torrent(ctx: &Arc<Ctx>, feed_id: String, enclosure: i64, url: String, d
#[tracing::instrument(skip(ctx))]
async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> {
let cfg = ctx.cfg();
let enc = ctx
.db
.enclosure(id).await?
.ok_or_else(|| anyhow::anyhow!("no enclosure {id}"))?;
let enc = ctx.db.enclosure(id).await?.ok_or_else(|| anyhow::anyhow!("no enclosure {id}"))?;
if enc.path.is_some() {
return Ok(()); // Already here.
}
@@ -2471,9 +2424,9 @@ mod tests {
let at = |last: Option<i64>, subscribed| {
due_at(&cfg, &f, &db::HttpState { last_checked: last, ..Default::default() }, subscribed)
};
assert_eq!(at(None, true), None); // never checked: due now
assert_eq!(at(Some(1000), true), Some(1000 + 3600)); // the hourly default
assert_eq!(at(Some(1000), false), Some(1000 + 86_400)); // listed, nobody subscribed
assert_eq!(at(None, true), None); // never checked: due now
assert_eq!(at(Some(1000), true), Some(1000 + 3600)); // the hourly default
assert_eq!(at(Some(1000), false), Some(1000 + 86_400)); // listed, nobody subscribed
}
#[test]
@@ -2496,14 +2449,20 @@ mod tests {
let base = empty_or_feed_server().await;
let ctx = Arc::new(test_ctx(config::Config::default()).await);
let cfg = config::Feed { url: format!("{base}/feed"), ..feed() };
assert!(matches!(scan_one(&ctx, "same", &cfg, &db::HttpState::default(), false, None).await, Ok(Outcome::Feed(_))));
assert!(matches!(
scan_one(&ctx, "same", &cfg, &db::HttpState::default(), false, None).await,
Ok(Outcome::Feed(_))
));
let state = ctx.db.http_state("same").await.unwrap();
let since = state.same_since.expect("when it was first read");
assert!(state.body_hash.is_some());
// Patreon's way: the whole feed again, no validator honoured (#242).
assert!(matches!(scan_one(&ctx, "same", &cfg, &state, false, None).await, Ok(Outcome::NotModified)));
assert_eq!(ctx.db.http_state("same").await.unwrap().same_since, Some(since), "still the same since then");
assert!(matches!(scan_one(&ctx, "same", &cfg, &state, true, None).await, Ok(Outcome::Feed(_))), "a forced check reads it");
assert!(
matches!(scan_one(&ctx, "same", &cfg, &state, true, None).await, Ok(Outcome::Feed(_))),
"a forced check reads it"
);
// Changed: read, and the same from now.
let moved = config::Feed { url: format!("{base}/fixed"), ..feed() };
assert!(matches!(scan_one(&ctx, "same", &moved, &state, false, None).await, Ok(Outcome::Feed(_))));
@@ -2614,11 +2573,7 @@ mod tests {
// Two filters: an item wanted by either of them is fetched, since one file serves
// both. The larger per-scan cap wins for the same reason.
let p = merge_policy(
&[sub(Some(&["rust"]), None, Some(2)), sub(Some(&["sqlite"]), None, Some(9))],
&feed(),
3,
);
let p = merge_policy(&[sub(Some(&["rust"]), None, Some(2)), sub(Some(&["sqlite"]), None, Some(9))], &feed(), 3);
assert!(p.wanted(&["rust"], &[]) && p.wanted(&["sqlite"], &[]));
assert!(!p.wanted(&["python"], &[]));
assert_eq!(p.budget, 9);
@@ -2692,19 +2647,34 @@ mod tests {
let (name, value) = line.rsplit_once(' ').unwrap();
assert!(name.starts_with("ipx_") && value.parse::<f64>().is_ok(), "{line}");
}
for name in ["ipx_db_connections{state=\"in_use\"}", "ipx_db_connections_max", "ipx_worker_queue", "ipx_runtime_tasks"] {
for name in
["ipx_db_connections{state=\"in_use\"}", "ipx_db_connections_max", "ipx_worker_queue", "ipx_runtime_tasks"]
{
assert!(text.contains(name), "{name}");
}
assert!(metrics::spawn(&at.to_string(), metrics::Sources {
ctx: ctx.clone(), cmds: mpsc::channel(1).0, events: broadcast::channel(1).0, playing: Default::default(),
runtime: tokio::runtime::Handle::current(), started: std::time::Instant::now(),
}).is_err(), "a port already taken is an error at start");
assert!(
metrics::spawn(
&at.to_string(),
metrics::Sources {
ctx: ctx.clone(),
cmds: mpsc::channel(1).0,
events: broadcast::channel(1).0,
playing: Default::default(),
runtime: tokio::runtime::Handle::current(),
started: std::time::Instant::now(),
}
)
.is_err(),
"a port already taken is an error at start"
);
}
#[tokio::test]
async fn dead_feeds_leave_the_directory_once_a_day_not_before_every_scan() {
let ctx = test_ctx(config::Config::default()).await;
let dead = |id: &str| format!("INSERT INTO feeds (id, url, error_since, managed) VALUES ('{id}', 'u-{id}', 100, false)");
let dead = |id: &str| {
format!("INSERT INTO feeds (id, url, error_since, managed) VALUES ('{id}', 'u-{id}', 100, false)")
};
let listed = async || ctx.db.feed_urls().await.unwrap().into_iter().map(|(id, _)| id).collect::<Vec<_>>();
ctx.db.sql(&dead("a")).await.unwrap();
clean_directory(&ctx).await.unwrap();
@@ -2817,8 +2787,13 @@ mod tests {
] {
db.sql(sql).await.unwrap();
}
db.store_plans(7, r#"{"300": {"cuts": [[60000,120000],[200000,260000]], "pieces": []},
"500": {"cuts": [[60000,100000]], "pieces": []}}"#).await.unwrap();
db.store_plans(
7,
r#"{"300": {"cuts": [[60000,120000],[200000,260000]], "pieces": []},
"500": {"cuts": [[60000,100000]], "pieces": []}}"#,
)
.await
.unwrap();
// One file on disk, three listeners at once: DeadAir at the usual pause, at a longer
// one, and the original.
let (usual, long, plain) = (
@@ -2965,7 +2940,9 @@ mod tests {
let got = feed::Fetched::Body { bytes: body, etag: None, last_modified: None };
let icon = "https://example.com/favicon.ico".to_string();
let cfg = config::Feed { url: "http://127.0.0.1:9/feed".into(), ..feed() };
scan_one(&ctx, "iconic", &cfg, &db::HttpState::default(), true, Some((Ok((got, None)), Some(icon.clone())))).await.unwrap();
scan_one(&ctx, "iconic", &cfg, &db::HttpState::default(), true, Some((Ok((got, None)), Some(icon.clone()))))
.await
.unwrap();
assert_eq!(ctx.db.feed_summary("iconic").await.unwrap().image, Some(icon));
}
@@ -3017,7 +2994,10 @@ mod tests {
let mut cfg = config::Config::default();
cfg.feeds.insert("opml".into(), feed());
let ctx = Arc::new(test_ctx(cfg).await);
let (me, you) = (ctx.db.create_user("me", None, false).await.unwrap(), ctx.db.create_user("you", None, false).await.unwrap());
let (me, you) = (
ctx.db.create_user("me", None, false).await.unwrap(),
ctx.db.create_user("you", None, false).await.unwrap(),
);
for id in ["opml", "a", "b"] {
if id != "opml" {
ctx.db.upsert_managed(id, &format!("http://x/{id}.xml"), id, "opml").await.unwrap();

259
crates/ipx/src/metrics.rs
View File

@@ -160,15 +160,16 @@ pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<std::net::SocketAddr> {
Err(e) => return tracing::error!(error = %e, "metrics could not start"),
};
rt.block_on(async move {
let app = axum::Router::new().route(
"/metrics",
axum::routing::get(move || {
let s = s.clone();
async move { ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(&s)) }
}),
)
// Prometheus asks for gzip, every 15 s (#253).
.layer(tower_http::compression::CompressionLayer::new().gzip(true));
let app = axum::Router::new()
.route(
"/metrics",
axum::routing::get(move || {
let s = s.clone();
async move { ([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(&s)) }
}),
)
// Prometheus asks for gzip, every 15 s (#253).
.layer(tower_http::compression::CompressionLayer::new().gzip(true));
match tokio::net::TcpListener::from_std(listener) {
Ok(l) => {
if let Err(e) = axum::serve(l, app).await {
@@ -186,24 +187,59 @@ pub fn spawn(bind: &str, s: Sources) -> anyhow::Result<std::net::SocketAddr> {
pub fn render(s: &Sources) -> String {
let mut out = String::new();
let (open, idle, max) = s.ctx.db.pool();
gauge(&mut out, "ipx_db_connections", "Database connections open, in use or idle.",
&[("state=\"in_use\"", open as f64 - idle as f64), ("state=\"idle\"", idle as f64)]);
gauge(
&mut out,
"ipx_db_connections",
"Database connections open, in use or idle.",
&[("state=\"in_use\"", open as f64 - idle as f64), ("state=\"idle\"", idle as f64)],
);
gauge(&mut out, "ipx_db_connections_max", "The most database connections the pool opens.", &[("", max as f64)]);
gauge(&mut out, "ipx_worker_queue", "Commands waiting for the worker: scans, downloads and the like.",
&[("", (s.cmds.max_capacity() - s.cmds.capacity()) as f64)]);
gauge(&mut out, "ipx_event_listeners", "Open pages and socket clients listening for live updates.",
&[("", s.events.receiver_count() as f64)]);
gauge(&mut out, "ipx_torrent_slots_free", "Torrents that could start now.",
&[("", s.ctx.torrent_slots.available_permits() as f64)]);
gauge(&mut out, "ipx_deadair_playing", "Episodes someone is playing through DeadAir's playlist.",
&[("", s.playing.lock().map_or(0, |p| p.len()) as f64)]);
work(&mut out, &STATS, s.events.len(), s.ctx.retry.lock().map_or(0, |r| r.len()), s.ctx.cfg().general.art_cache_mb * 1_048_576);
gauge(&mut out, "ipx_signals_keys_refused", "People whose TypeSafe key was refused, whose Signals wait until they save another.",
&[("", s.ctx.signals_refused.lock().map_or(0, |r| r.len()) as f64)]);
gauge(
&mut out,
"ipx_worker_queue",
"Commands waiting for the worker: scans, downloads and the like.",
&[("", (s.cmds.max_capacity() - s.cmds.capacity()) as f64)],
);
gauge(
&mut out,
"ipx_event_listeners",
"Open pages and socket clients listening for live updates.",
&[("", s.events.receiver_count() as f64)],
);
gauge(
&mut out,
"ipx_torrent_slots_free",
"Torrents that could start now.",
&[("", s.ctx.torrent_slots.available_permits() as f64)],
);
gauge(
&mut out,
"ipx_deadair_playing",
"Episodes someone is playing through DeadAir's playlist.",
&[("", s.playing.lock().map_or(0, |p| p.len()) as f64)],
);
work(
&mut out,
&STATS,
s.events.len(),
s.ctx.retry.lock().map_or(0, |r| r.len()),
s.ctx.cfg().general.art_cache_mb * 1_048_576,
);
gauge(
&mut out,
"ipx_signals_keys_refused",
"People whose TypeSafe key was refused, whose Signals wait until they save another.",
&[("", s.ctx.signals_refused.lock().map_or(0, |r| r.len()) as f64)],
);
let rt = s.runtime.metrics();
gauge(&mut out, "ipx_runtime_workers", "The async runtime's worker threads.", &[("", rt.num_workers() as f64)]);
gauge(&mut out, "ipx_runtime_tasks", "Tasks alive in the async runtime.", &[("", rt.num_alive_tasks() as f64)]);
gauge(&mut out, "ipx_runtime_queue", "Tasks waiting in the runtime's global queue.", &[("", rt.global_queue_depth() as f64)]);
gauge(
&mut out,
"ipx_runtime_queue",
"Tasks waiting in the runtime's global queue.",
&[("", rt.global_queue_depth() as f64)],
);
let (rss, fds) = ipx_directory::metrics::process();
if let Some(v) = rss {
gauge(&mut out, "ipx_process_resident_bytes", "Memory the process holds.", &[("", v)]);
@@ -211,9 +247,24 @@ pub fn render(s: &Sources) -> String {
if let Some(v) = fds {
gauge(&mut out, "ipx_process_open_fds", "Files and sockets the process has open.", &[("", v)]);
}
counter(&mut out, "ipx_feed_fetches_total", "Feeds fetched, answered 304 included.", &[("", ipx_feed::FETCHES.load(Relaxed) as f64)]);
counter(&mut out, "ipx_feed_bytes_total", "Bytes of feed bodies fetched.", &[("", ipx_feed::FETCHED_BYTES.load(Relaxed) as f64)]);
gauge(&mut out, "ipx_uptime_seconds", "Seconds since the daemon started.", &[("", s.started.elapsed().as_secs() as f64)]);
counter(
&mut out,
"ipx_feed_fetches_total",
"Feeds fetched, answered 304 included.",
&[("", ipx_feed::FETCHES.load(Relaxed) as f64)],
);
counter(
&mut out,
"ipx_feed_bytes_total",
"Bytes of feed bodies fetched.",
&[("", ipx_feed::FETCHED_BYTES.load(Relaxed) as f64)],
);
gauge(
&mut out,
"ipx_uptime_seconds",
"Seconds since the daemon started.",
&[("", s.started.elapsed().as_secs() as f64)],
);
let version = format!("version=\"{}\"", env!("CARGO_PKG_VERSION"));
gauge(&mut out, "ipx_build_info", "The version running, as a label.", &[(&version, 1.0)]);
out
@@ -226,54 +277,139 @@ fn work(out: &mut String, st: &Stats, event_queue: usize, retries: usize, art_li
gauge(out, "ipx_worker_busy", "1 while the worker is on a job.", &[("", job.is_some() as u8 as f64)]);
if let Some((kind, since)) = job {
let label = format!("kind=\"{kind}\"");
gauge(out, "ipx_worker_job_seconds", "How long the worker has been on its job, by the job's kind.",
&[(&label, since.elapsed().as_secs_f64())]);
gauge(
out,
"ipx_worker_job_seconds",
"How long the worker has been on its job, by the job's kind.",
&[(&label, since.elapsed().as_secs_f64())],
);
}
let jobs = st.jobs.lock().unwrap().clone();
let labels: Vec<String> = jobs.keys().map(|k| format!("kind=\"{k}\"")).collect();
let pairs = |f: fn(&(u64, f64)) -> f64| labels.iter().zip(jobs.values()).map(|(l, v)| (l.as_str(), f(v))).collect::<Vec<_>>();
let pairs = |f: fn(&(u64, f64)) -> f64| {
labels.iter().zip(jobs.values()).map(|(l, v)| (l.as_str(), f(v))).collect::<Vec<_>>()
};
counter(out, "ipx_jobs_total", "Jobs the worker has finished, by kind.", &pairs(|v| v.0 as f64));
counter(out, "ipx_job_seconds_total", "Seconds the worker has spent on jobs, by kind.", &pairs(|v| v.1));
let last = st.last_scan_done.load(Relaxed);
if last > 0 {
gauge(out, "ipx_last_scan_age_seconds", "Seconds since a scan last finished.", &[("", (now - last) as f64)]);
}
gauge(out, "ipx_http_requests_in_flight", "Web requests being answered.", &[("", st.http_in_flight.load(Relaxed) as f64)]);
gauge(out, "ipx_scan_feeds_fetching", "Feeds a scan is fetching at once.", &[("", st.feeds_fetching.load(Relaxed) as f64)]);
gauge(
out,
"ipx_http_requests_in_flight",
"Web requests being answered.",
&[("", st.http_in_flight.load(Relaxed) as f64)],
);
gauge(
out,
"ipx_scan_feeds_fetching",
"Feeds a scan is fetching at once.",
&[("", st.feeds_fetching.load(Relaxed) as f64)],
);
gauge(out, "ipx_downloads_active", "Files downloading now.", &[("", st.downloads_active.load(Relaxed) as f64)]);
counter(out, "ipx_download_bytes_total", "Bytes downloaded since the daemon started.", &[("", st.download_bytes.load(Relaxed) as f64)]);
counter(
out,
"ipx_download_bytes_total",
"Bytes downloaded since the daemon started.",
&[("", st.download_bytes.load(Relaxed) as f64)],
);
gauge(out, "ipx_event_queue", "Live updates sent and not yet read by every listener.", &[("", event_queue as f64)]);
counter(out, "ipx_events_missed_total", "Live updates dropped for a listener that fell behind.", &[("", st.events_missed.load(Relaxed) as f64)]);
gauge(out, "ipx_deadair_backlog", "Files of shows someone has DeadAir on that it has not looked at, those already heard through included.", &[("", st.deadair_backlog.load(Relaxed) as f64)]);
counter(
out,
"ipx_events_missed_total",
"Live updates dropped for a listener that fell behind.",
&[("", st.events_missed.load(Relaxed) as f64)],
);
gauge(
out,
"ipx_deadair_backlog",
"Files of shows someone has DeadAir on that it has not looked at, those already heard through included.",
&[("", st.deadair_backlog.load(Relaxed) as f64)],
);
let since = st.deadair_busy_since.load(Relaxed);
gauge(out, "ipx_deadair_busy_seconds", "How long DeadAir has been on its file; 0 when on none.",
&[("", if since > 0 { (now - since) as f64 } else { 0.0 })]);
gauge(
out,
"ipx_deadair_busy_seconds",
"How long DeadAir has been on its file; 0 when on none.",
&[("", if since > 0 { (now - since) as f64 } else { 0.0 })],
);
gauge(out, "ipx_feed_retries", "Feeds waiting to try a failed name lookup again.", &[("", retries as f64)]);
gauge(out, "ipx_art_cache_bytes", "Artwork kept on disk, as of the last sweep.", &[("", st.art_cache_bytes.load(Relaxed) as f64)]);
gauge(
out,
"ipx_art_cache_bytes",
"Artwork kept on disk, as of the last sweep.",
&[("", st.art_cache_bytes.load(Relaxed) as f64)],
);
gauge(out, "ipx_art_cache_limit_bytes", "The most artwork kept.", &[("", art_limit as f64)]);
counter(out, "ipx_art_evicted_total", "Images thrown out to keep the artwork under its limit.", &[("", st.art_evicted.load(Relaxed) as f64)]);
counter(out, "ipx_art_requests_total", "Artwork asked for: served from disk, or fetched from the publisher.",
&[("result=\"hit\"", st.art_hits.load(Relaxed) as f64), ("result=\"miss\"", st.art_misses.load(Relaxed) as f64)]);
counter(
out,
"ipx_art_evicted_total",
"Images thrown out to keep the artwork under its limit.",
&[("", st.art_evicted.load(Relaxed) as f64)],
);
counter(
out,
"ipx_art_requests_total",
"Artwork asked for: served from disk, or fetched from the publisher.",
&[
("result=\"hit\"", st.art_hits.load(Relaxed) as f64),
("result=\"miss\"", st.art_misses.load(Relaxed) as f64),
],
);
// By person and whose key paid (#278). An account's name is often an email address: /metrics
// is on the LAN only, as Prometheus is.
let people = st.people.lock().unwrap().clone();
let labels: Vec<String> = people.keys().map(|(who, key)| format!("user=\"{}\",key=\"{key}\"", label(who))).collect();
let labels: Vec<String> =
people.keys().map(|(who, key)| format!("user=\"{}\",key=\"{key}\"", label(who))).collect();
let by = |n: usize| labels.iter().zip(people.values()).map(|(l, v)| (l.as_str(), v[n])).collect::<Vec<_>>();
counter(out, "ipx_signals_judged_total", "Items Jev has judged for someone's Signals, by person and whose key paid.", &by(0));
counter(
out,
"ipx_signals_judged_total",
"Items Jev has judged for someone's Signals, by person and whose key paid.",
&by(0),
);
counter(out, "ipx_signals_hits_total", "Signals an item was found to fit, by person and whose key paid.", &by(1));
let tokens: Vec<String> = labels.iter().flat_map(|l| [format!("kind=\"input\",{l}"), format!("kind=\"output\",{l}")]).collect();
let used: Vec<(&str, f64)> = tokens.iter().zip(people.values().flat_map(|v| [v[2], v[3]])).map(|(l, v)| (l.as_str(), v)).collect();
counter(out, "ipx_jev_tokens_total", "Tokens TypeSafe counted for calls to Jev, what they cost, by person and whose key paid.", &used);
let tokens: Vec<String> =
labels.iter().flat_map(|l| [format!("kind=\"input\",{l}"), format!("kind=\"output\",{l}")]).collect();
let used: Vec<(&str, f64)> =
tokens.iter().zip(people.values().flat_map(|v| [v[2], v[3]])).map(|(l, v)| (l.as_str(), v)).collect();
counter(
out,
"ipx_jev_tokens_total",
"Tokens TypeSafe counted for calls to Jev, what they cost, by person and whose key paid.",
&used,
);
let last = st.signals_last_pass.load(Relaxed);
if last > 0 {
gauge(out, "ipx_signals_last_pass_age_seconds", "Seconds since the Signals judge last finished a pass.", &[("", (now - last) as f64)]);
gauge(
out,
"ipx_signals_last_pass_age_seconds",
"Seconds since the Signals judge last finished a pass.",
&[("", (now - last) as f64)],
);
}
let labels: Vec<String> = JEV_RESULTS.iter().map(|r| format!("result=\"{r}\"")).collect();
let calls: Vec<(&str, f64)> = labels.iter().zip(&st.jev_calls).map(|(l, c)| (l.as_str(), c.load(Relaxed) as f64)).collect();
counter(out, "ipx_jev_requests_total", "Calls to TypeSafe's Jev, Signals' and their icons', by how they ended.", &calls);
counter(out, "ipx_jev_seconds_total", "Seconds spent on calls to Jev, their wait for a turn included.", &[("", st.jev_micros.load(Relaxed) as f64 / 1e6)]);
counter(out, "ipx_jev_paced_seconds_total", "Seconds calls to Jev waited for their turn under TypeSafe's rate limits.",
&[("", st.jev_paced_micros.load(Relaxed) as f64 / 1e6)]);
let calls: Vec<(&str, f64)> =
labels.iter().zip(&st.jev_calls).map(|(l, c)| (l.as_str(), c.load(Relaxed) as f64)).collect();
counter(
out,
"ipx_jev_requests_total",
"Calls to TypeSafe's Jev, Signals' and their icons', by how they ended.",
&calls,
);
counter(
out,
"ipx_jev_seconds_total",
"Seconds spent on calls to Jev, their wait for a turn included.",
&[("", st.jev_micros.load(Relaxed) as f64 / 1e6)],
);
counter(
out,
"ipx_jev_paced_seconds_total",
"Seconds calls to Jev waited for their turn under TypeSafe's rate limits.",
&[("", st.jev_paced_micros.load(Relaxed) as f64 / 1e6)],
);
for (name, help, v) in [
("ipx_feeds", "Feeds in the catalogue, as last counted.", &st.feeds),
("ipx_downloads_pending", "Files a scan will download on its own, as last counted.", &st.pending),
@@ -336,23 +472,36 @@ mod tests {
assert_eq!(T.downloads_active.load(Relaxed), 1);
let mut out = String::new();
work(&mut out, &T, 0, 0, 0);
assert!(out.contains("ipx_worker_busy 1\n") && out.contains("ipx_worker_job_seconds{kind=\"scan\"}"), "{out}");
assert!(
out.contains("ipx_worker_busy 1\n") && out.contains("ipx_worker_job_seconds{kind=\"scan\"}"),
"{out}"
);
}
// Dropped, as an error or a stop drops them: nothing left in progress, one scan done.
assert_eq!(T.downloads_active.load(Relaxed), 0);
let mut out = String::new();
work(&mut out, &T, 3, 1, 10);
assert!(out.contains("ipx_worker_busy 0\n") && !out.contains("ipx_worker_job_seconds"), "{out}");
assert!(out.contains("ipx_jobs_total{kind=\"scan\"} 1\n") && out.contains("# TYPE ipx_jobs_total counter"), "{out}");
assert!(
out.contains("ipx_jobs_total{kind=\"scan\"} 1\n") && out.contains("# TYPE ipx_jobs_total counter"),
"{out}"
);
assert!(out.contains("ipx_event_queue 3\n") && out.contains("ipx_feed_retries 1\n"));
assert!(out.contains("ipx_jev_requests_total{result=\"key_refused\"} 0\n") && !out.contains("ipx_signals_last_pass_age_seconds"), "{out}");
assert!(
out.contains("ipx_jev_requests_total{result=\"key_refused\"} 0\n")
&& !out.contains("ipx_signals_last_pass_age_seconds"),
"{out}"
);
T.bill("sam@example.net", "server", [1.0, 1.0, 600.0, 50.0]);
T.bill("sam@example.net", "server", [1.0, 0.0, 300.5, 25.0]);
T.bill("a \"b\"", "own", [0.0, 0.0, 10.0, 1.0]);
let mut out = String::new();
work(&mut out, &T, 0, 0, 0);
assert!(out.contains("ipx_signals_judged_total{user=\"sam@example.net\",key=\"server\"} 2\n"), "{out}");
assert!(out.contains("ipx_jev_tokens_total{kind=\"input\",user=\"sam@example.net\",key=\"server\"} 900.5\n"), "{out}");
assert!(
out.contains("ipx_jev_tokens_total{kind=\"input\",user=\"sam@example.net\",key=\"server\"} 900.5\n"),
"{out}"
);
assert!(out.contains("ipx_jev_tokens_total{kind=\"output\",user=\"a \\\"b\\\"\",key=\"own\"} 1\n"), "{out}");
// The library's counts are left out until the daemon has counted it.
assert!(!out.contains("ipx_feeds "));

36
crates/ipx/src/retention.rs
View File

@@ -39,11 +39,7 @@ pub fn pick(candidates: &[Candidate], total: u64, limit: u64) -> Vec<Candidate>
/// Files older than the cutoff. Flagged ones are already excluded by the query.
pub fn aged(candidates: &[Candidate], cutoff: i64) -> Vec<Candidate> {
candidates
.iter()
.filter(|c| c.age_key > 0 && c.age_key < cutoff)
.cloned()
.collect()
candidates.iter().filter(|c| c.age_key > 0 && c.age_key < cutoff).cloned().collect()
}
pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
@@ -79,11 +75,8 @@ pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
if cfg.general.max_total_gb > 0.0 {
let limit = (cfg.general.max_total_gb * 1_073_741_824.0) as u64;
// Re-read: the age sweep may already have freed enough.
let remaining: Vec<Candidate> = candidates
.iter()
.filter(|c| !report.aged_out.iter().any(|a| a.id == c.id))
.cloned()
.collect();
let remaining: Vec<Candidate> =
candidates.iter().filter(|c| !report.aged_out.iter().any(|a| a.id == c.id)).cloned().collect();
let total: u64 = remaining.iter().map(|c| c.bytes.max(0) as u64).sum();
report.over_quota = pick(&remaining, total, limit);
for c in &report.over_quota {
@@ -99,10 +92,7 @@ async fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result<u64> {
return Ok(c.bytes.max(0) as u64);
}
// A torrent's folder measures as a directory entry, not what is in it: the recorded size then.
let size = std::fs::metadata(&c.path)
.ok()
.filter(|m| m.is_file())
.map_or(c.bytes.max(0) as u64, |m| m.len());
let size = std::fs::metadata(&c.path).ok().filter(|m| m.is_file()).map_or(c.bytes.max(0) as u64, |m| m.len());
// Missing is fine: the point is that it is gone.
if let Err(e) = crate::download::remove(std::path::Path::new(&c.path))
&& e.kind() != std::io::ErrorKind::NotFound
@@ -119,14 +109,7 @@ mod tests {
use super::*;
fn cand(id: i64, bytes: i64, age_key: i64, read: bool) -> Candidate {
Candidate {
id,
url: format!("https://x/{id}"),
path: format!("/tmp/{id}"),
bytes,
age_key,
read,
}
Candidate { id, url: format!("https://x/{id}"), path: format!("/tmp/{id}"), bytes, age_key, read }
}
#[test]
@@ -183,7 +166,8 @@ mod tests {
(2, 'f', 'half', 'u2', '/tmp/half', 10, 'done', 20),
(3, 'f', 'unread', 'u3', '/tmp/unread', 10, 'done', 30),
(4, 'f', 'read', 'u4', '/tmp/read', 10, 'done', 40);",
).await
)
.await
.unwrap();
let got: Vec<i64> = db.reap_candidates().await.unwrap().iter().map(|c| c.id).collect();
@@ -212,7 +196,8 @@ mod tests {
INSERT INTO enclosures (id, feed_id, guid, url, path, bytes_done, state, downloaded_at) VALUES
(1, 'f', 'heard', 'u1', '/tmp/heard', 10, 'done', 10),
(2, 'f', 'pinned', 'u2', '/tmp/pinned', 10, 'done', 20);",
).await
)
.await
.unwrap();
let got: Vec<(i64, bool)> = db.reap_candidates().await.unwrap().iter().map(|c| (c.id, c.read)).collect();
assert_eq!(got, vec![(1, false), (2, false)], "sam has not heard it, and nobody here pinned it (#156)");
@@ -233,7 +218,8 @@ mod tests {
('f', 'recent', 900);
INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES
(1, 'f', 'has-file', 'u1', '/tmp/x', 'done');",
).await
)
.await
.unwrap();
assert_eq!(db.prune_entries(500).await.unwrap(), 1, "only the old, fileless, unflagged one");

110
crates/ipx/src/signals.rs
View File

@@ -79,13 +79,20 @@ pub fn typesafe_key() -> Option<String> {
async fn pass(ctx: &Ctx) -> Result<bool> {
let server = server_key(ctx);
// People's refused keys are not what is asked with when the server's is.
let skip: Vec<i64> = if server.is_some() { Vec::new() } else { ctx.signals_refused.lock().unwrap().iter().copied().collect() };
let skip: Vec<i64> =
if server.is_some() { Vec::new() } else { ctx.signals_refused.lock().unwrap().iter().copied().collect() };
let days = ctx.cfg().general.signal_days;
let items = ctx.db.unjudged(&skip, BATCH, server.is_some(), days).await?;
if items.is_empty() {
return Ok(false);
}
let span = tracing::info_span!("signals", entries = items.len(), calls = tracing::field::Empty, judged = tracing::field::Empty, hits = tracing::field::Empty);
let span = tracing::info_span!(
"signals",
entries = items.len(),
calls = tracing::field::Empty,
judged = tracing::field::Empty,
hits = tracing::field::Empty
);
async {
let groups = calls(&items, server.is_some());
let paid = if server.is_some() { "server" } else { "own" };
@@ -184,7 +191,12 @@ fn calls(items: &[Unjudged], share: bool) -> Vec<Vec<&Unjudged>> {
let mut out: Vec<Vec<&Unjudged>> = Vec::new();
for item in items {
match out.last_mut() {
Some(g) if share && (g[0].feed_id.as_str(), g[0].guid.as_str()) == (item.feed_id.as_str(), item.guid.as_str()) => g.push(item),
Some(g)
if share
&& (g[0].feed_id.as_str(), g[0].guid.as_str()) == (item.feed_id.as_str(), item.guid.as_str()) =>
{
g.push(item)
}
_ => out.push(vec![item]),
}
}
@@ -228,7 +240,11 @@ fn rate_limited(e: &anyhow::Error) -> bool {
/// A call's turn: it starts at `next` or now, whichever is later, and holds the line for its
/// share of a second under both limits. Gives when it starts and when the one after may.
fn slot(next: tokio::time::Instant, now: tokio::time::Instant, tokens: f64) -> (tokio::time::Instant, tokio::time::Instant) {
fn slot(
next: tokio::time::Instant,
now: tokio::time::Instant,
tokens: f64,
) -> (tokio::time::Instant, tokio::time::Instant) {
let start = next.max(now);
(start, start + std::time::Duration::from_secs_f64((1.0 / MAX_RATE).max(tokens / MAX_TOKENS)))
}
@@ -297,8 +313,7 @@ fn request(item: &Unjudged, asks: &[&Ask]) -> serde_json::Value {
item.feed_title.as_deref().unwrap_or(""),
item.title.as_deref().unwrap_or("")
);
let questions: serde_json::Map<String, serde_json::Value> =
asks.iter().map(|a| (key(a.id), question(a))).collect();
let questions: serde_json::Map<String, serde_json::Value> = asks.iter().map(|a| (key(a.id), question(a))).collect();
serde_json::json!({ "model": "jev-latest", "state": state, "questions": questions })
}
@@ -312,7 +327,11 @@ fn key(id: i64) -> String {
/// 0.02, with AI code review still 0.94 (#259). `criteria` is an object of what true and false
/// mean: a sentence on its own was refused with a 422 (#257).
fn question(a: &Ask) -> serde_json::Value {
let mut said = format!("The reader follows a subject they call \"{}\", described by these words: {}.", a.name, a.keywords.join(", "));
let mut said = format!(
"The reader follows a subject they call \"{}\", described by these words: {}.",
a.name,
a.keywords.join(", ")
);
if let Some(d) = &a.description {
said.push_str(&format!(" In their own words: {d}"));
}
@@ -335,7 +354,15 @@ fn hit(answer: &serde_json::Value, id: i64) -> Option<bool> {
/// chooses among options and cannot name one, so the page searches Font Awesome, whose index is
/// ipx-web's, and sends the likeliest. On the server's key, not the Signal owner's: it is the
/// server's page doing it, once a Signal.
pub async fn pick_icon(ctx: &Ctx, who: &str, key: &str, name: &str, words: &[String], description: Option<&str>, candidates: &[(String, String)]) -> Result<String> {
pub async fn pick_icon(
ctx: &Ctx,
who: &str,
key: &str,
name: &str,
words: &[String],
description: Option<&str>,
candidates: &[(String, String)],
) -> Result<String> {
let span = tracing::info_span!(
"jev_icon",
candidates = candidates.len(),
@@ -439,7 +466,12 @@ fn outcome<T>(r: &Result<T>) -> usize {
}
}
fn icon_request(name: &str, words: &[String], description: Option<&str>, candidates: &[(String, String)]) -> serde_json::Value {
fn icon_request(
name: &str,
words: &[String],
description: Option<&str>,
candidates: &[(String, String)],
) -> serde_json::Value {
let mut state = format!("Signal: {name}\nListens for: {}", words.join(", "));
if let Some(d) = description {
state.push_str(&format!("\nIn their words: {d}"));
@@ -522,7 +554,10 @@ mod tests {
#[test]
fn an_icon_is_one_jev_was_offered() {
let offered = vec![("solid/robot".to_string(), "Robot: ai, bot".to_string()), ("solid/laptop-code".to_string(), "Laptop Code".to_string())];
let offered = vec![
("solid/robot".to_string(), "Robot: ai, bot".to_string()),
("solid/laptop-code".to_string(), "Laptop Code".to_string()),
];
let r = icon_request("AI Development", &["ai".into()], Some("AI for writing software."), &offered);
assert_eq!(r["questions"]["icon"]["type"], "choice");
assert_eq!(r["questions"]["icon"]["criteria"]["solid/robot"], "Robot: ai, bot");
@@ -557,11 +592,20 @@ mod tests {
#[test]
fn on_the_servers_key_one_item_is_one_call_for_everyone() {
let row = |user_id, guid: &str| Unjudged {
user_id, user_name: format!("u{user_id}"), key: None, feed_id: "f".into(), guid: guid.into(), feed_title: None, title: None, description: None,
user_id,
user_name: format!("u{user_id}"),
key: None,
feed_id: "f".into(),
guid: guid.into(),
feed_title: None,
title: None,
description: None,
signals: vec![Ask { id: user_id * 10, name: "s".into(), keywords: vec![], description: None }],
};
let items = [row(1, "x"), row(2, "x"), row(1, "y")];
let shape = |share| calls(&items, share).iter().map(|g| g.iter().map(|i| i.user_id).collect::<Vec<_>>()).collect::<Vec<_>>();
let shape = |share| {
calls(&items, share).iter().map(|g| g.iter().map(|i| i.user_id).collect::<Vec<_>>()).collect::<Vec<_>>()
};
assert_eq!(shape(true), [vec![1, 2], vec![1]], "x asked once, for both");
assert_eq!(shape(false), [vec![1], vec![2], vec![1]], "each on their own key");
let both: Vec<&Ask> = calls(&items, true)[0].iter().flat_map(|i| &i.signals).collect();
@@ -587,10 +631,22 @@ mod tests {
#[test]
fn a_batch_is_full_by_its_rows_not_its_items() {
let item = |n: usize| Unjudged {
user_id: 1, user_name: "a".into(), key: Some("k".into()), feed_id: "f".into(), guid: "g".into(), feed_title: None, title: None, description: None,
signals: (0..n as i64).map(|id| Ask { id, name: "s".into(), keywords: vec![], description: None }).collect(),
user_id: 1,
user_name: "a".into(),
key: Some("k".into()),
feed_id: "f".into(),
guid: "g".into(),
feed_title: None,
title: None,
description: None,
signals: (0..n as i64)
.map(|id| Ask { id, name: "s".into(), keywords: vec![], description: None })
.collect(),
};
assert!(full(&(0..25).map(|_| item(2)).collect::<Vec<_>>()), "25 items of two Signals each is the 50 asked for");
assert!(
full(&(0..25).map(|_| item(2)).collect::<Vec<_>>()),
"25 items of two Signals each is the 50 asked for"
);
assert!(!full(&(0..24).map(|_| item(2)).collect::<Vec<_>>()));
}
@@ -607,7 +663,12 @@ mod tests {
description: Some("<p>Qualifying <b>report</b></p>".into()),
signals: vec![
Ask { id: 3, name: "F1".into(), keywords: vec!["formula 1".into(), "f1".into()], description: None },
Ask { id: 4, name: "Food".into(), keywords: vec!["cooking".into()], description: Some("Recipes, not restaurants.".into()) },
Ask {
id: 4,
name: "Food".into(),
keywords: vec!["cooking".into()],
description: Some("Recipes, not restaurants.".into()),
},
],
};
let r = request(&item, &item.signals.iter().collect::<Vec<_>>());
@@ -615,9 +676,20 @@ mod tests {
let q = &r["questions"]["signal_3"];
assert_eq!(q["type"], "noul");
let says = q["instructions"].as_str().unwrap();
assert!(says.contains("\"F1\"") && says.contains("formula 1, f1"), "one subject, by its name and its words: {says}");
assert!(
says.contains("\"F1\"") && says.contains("formula 1, f1"),
"one subject, by its name and its words: {says}"
);
assert!(!says.contains("own words"), "no sentence, none said");
assert!(q["criteria"]["true"].is_string() && q["criteria"]["false"].is_string(), "TypeSafe takes an object, not a sentence");
assert!(r["questions"]["signal_4"]["instructions"].as_str().unwrap().contains("In their own words: Recipes, not restaurants."));
assert!(
q["criteria"]["true"].is_string() && q["criteria"]["false"].is_string(),
"TypeSafe takes an object, not a sentence"
);
assert!(
r["questions"]["signal_4"]["instructions"]
.as_str()
.unwrap()
.contains("In their own words: Recipes, not restaurants.")
);
}
}

17
crates/ipx/src/torrent.rs
View File

@@ -26,10 +26,7 @@ impl Torrents {
let listen_addr: SocketAddr = format!("0.0.0.0:{lo}").parse()?;
let opts = SessionOptions {
listen: Some(librqbit::ListenerOptions {
listen_addr,
..Default::default()
}),
listen: Some(librqbit::ListenerOptions { listen_addr, ..Default::default() }),
client_name_and_version: Some(concat!("ipx ", env!("CARGO_PKG_VERSION")).into()),
..Default::default()
};
@@ -72,12 +69,7 @@ impl Torrents {
),
)
.await
.map_err(|_| {
anyhow::anyhow!(
"no metadata after {} minutes, gave up",
limit.as_secs() / 60
)
})?
.map_err(|_| anyhow::anyhow!("no metadata after {} minutes, gave up", limit.as_secs() / 60))?
.context("adding the torrent")?;
let (id, handle) = match added {
@@ -116,10 +108,7 @@ impl Torrents {
let size = handle.stats().total_bytes;
self.seed(cfg, &handle).await;
self.session
.delete(id.into(), false)
.await
.context("releasing the torrent")?;
self.session.delete(id.into(), false).await.context("releasing the torrent")?;
Ok((path, size))
}

386
crates/ipx/src/web.rs
View File

@@ -14,16 +14,16 @@ use axum::{
routing::{delete, get, patch, post},
};
use serde::Deserialize;
use tower::ServiceExt;
use tower_http::services::ServeFile;
use serde::Serialize;
use std::sync::Arc;
use tokio::sync::{broadcast, mpsc};
use tower::ServiceExt;
use tower_http::services::ServeFile;
use crate::Ctx;
use crate::db::step;
use tracing::Instrument;
use crate::ipc::{Command, Event};
use tracing::Instrument;
const COOKIE: &str = "ipx_token";
@@ -97,21 +97,16 @@ pub async fn serve(state: WebState, bind: &str) -> Result<()> {
use axum::serve::ListenerExt;
// Nagle off, as axum 0.7 had it: under load a response goes out in parts, and with Nagle
// on its last small part waits for the browser's delayed ACK, up to 40 ms (#170).
let listener = tokio::net::TcpListener::bind(bind)
.await
.with_context(|| format!("binding {bind}"))?
.tap_io(|tcp| {
let listener =
tokio::net::TcpListener::bind(bind).await.with_context(|| format!("binding {bind}"))?.tap_io(|tcp| {
if let Err(e) = tcp.set_nodelay(true) {
tracing::debug!(error = %e, "could not set TCP_NODELAY");
}
});
tracing::info!(bind, "web ui listening");
axum::serve(
listener,
router(state).into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.await
.context("serving the web ui")
axum::serve(listener, router(state).into_make_service_with_connect_info::<std::net::SocketAddr>())
.await
.context("serving the web ui")
}
/// Who is asking, in order of how specific the claim is: a proxy that vouches for a name, an
@@ -181,20 +176,14 @@ async fn auth(State(state): State<WebState>, mut req: Request, next: Next) -> Re
// 2. A session cookie from signing in here.
if user.is_none() {
if let Some(sid) = cookie(&req, SESSION_COOKIE) {
user = state
.ctx
.db
.session_user(&sid, cfg.web.session_days.max(1) * 86_400).await
.unwrap_or(None);
user = state.ctx.db.session_user(&sid, cfg.web.session_days.max(1) * 86_400).await.unwrap_or(None);
}
}
// 3. The shared token, which is the admin: the healthcheck and any scripts predate
// accounts and must keep working.
let from_query = req.uri().query().and_then(|q| {
q.split('&')
.find_map(|kv| kv.strip_prefix("token=").map(str::to_owned))
});
let from_query =
req.uri().query().and_then(|q| q.split('&').find_map(|kv| kv.strip_prefix("token=").map(str::to_owned)));
if user.is_none() && !token.is_empty() {
let supplied = from_query.clone().or_else(|| cookie(&req, COOKIE));
if supplied.is_some_and(|t| constant_time_eq(&t, &token)) {
@@ -204,9 +193,7 @@ async fn auth(State(state): State<WebState>, mut req: Request, next: Next) -> Re
if let Some(u) = &user {
let _ = state.ctx.db.signed_in(u.id).await;
}
set_cookie = Some(format!(
"{COOKIE}={token}; Path=/; HttpOnly; SameSite=Lax; Max-Age=31536000"
));
set_cookie = Some(format!("{COOKIE}={token}; Path=/; HttpOnly; SameSite=Lax; Max-Age=31536000"));
}
}
}
@@ -214,11 +201,8 @@ async fn auth(State(state): State<WebState>, mut req: Request, next: Next) -> Re
let Some(user) = user else {
// A browser asking for a page gets the sign-in form; anything else gets a 401 it
// can act on.
let wants_html = req
.headers()
.get(header::ACCEPT)
.and_then(|v| v.to_str().ok())
.is_some_and(|a| a.contains("text/html"));
let wants_html =
req.headers().get(header::ACCEPT).and_then(|v| v.to_str().ok()).is_some_and(|a| a.contains("text/html"));
return if wants_html {
Redirect::to("/login").into_response()
} else {
@@ -251,7 +235,12 @@ struct Proxied(bool);
///
/// Takes the request's parts rather than the request: a `&Request` held across the await makes
/// the future unsendable, as a body is not `Sync`.
async fn vouched_name(state: &WebState, cfg: &crate::config::Config, peer: String, headers: &axum::http::HeaderMap) -> Option<String> {
async fn vouched_name(
state: &WebState,
cfg: &crate::config::Config,
peer: String,
headers: &axum::http::HeaderMap,
) -> Option<String> {
if cfg.web.trusted_header.is_empty() || !cfg.web.trusted_proxies.iter().any(|p| p == &peer) {
return None;
}
@@ -282,11 +271,7 @@ impl<S: Send + Sync> axum::extract::FromRequestParts<S> for crate::db::User {
parts: &mut axum::http::request::Parts,
_state: &S,
) -> std::result::Result<Self, Self::Rejection> {
parts
.extensions
.get::<crate::db::User>()
.cloned()
.ok_or((StatusCode::UNAUTHORIZED, "sign in"))
parts.extensions.get::<crate::db::User>().cloned().ok_or((StatusCode::UNAUTHORIZED, "sign in"))
}
}
@@ -298,20 +283,13 @@ fn headers_cookie(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(header::COOKIE)
.and_then(|v| v.to_str().ok())
.and_then(|c| {
c.split(';')
.find_map(|kv| kv.trim().strip_prefix(&format!("{name}=")).map(str::to_owned))
})
.and_then(|c| c.split(';').find_map(|kv| kv.trim().strip_prefix(&format!("{name}=")).map(str::to_owned)))
}
/// The account the shared token stands for: the first admin, or the first user at all.
async fn admin_user(state: &WebState) -> Option<crate::db::User> {
let users = state.ctx.db.users().await.ok()?;
users
.iter()
.find(|u| u.is_admin)
.or_else(|| users.first())
.cloned()
users.iter().find(|u| u.is_admin).or_else(|| users.first()).cloned()
}
#[derive(Deserialize)]
@@ -320,10 +298,7 @@ struct Credentials {
password: String,
}
async fn login(
State(state): State<WebState>,
Json(body): Json<Credentials>,
) -> Result<Response, ApiError> {
async fn login(State(state): State<WebState>, Json(body): Json<Credentials>) -> Result<Response, ApiError> {
let name = body.name.trim().to_ascii_lowercase();
let user = state.ctx.db.user_by_name(&name).await?;
// The same answer either way, in the same time: whether a name exists is not something to leak.
@@ -340,8 +315,7 @@ async fn login(
tracing::info!(user = %user.name, "signed in");
let days = state.ctx.cfg().web.session_days.max(1);
let mut resp = Json(serde_json::json!({ "name": user.name, "admin": user.is_admin }))
.into_response();
let mut resp = Json(serde_json::json!({ "name": user.name, "admin": user.is_admin })).into_response();
if let Ok(v) = header::HeaderValue::from_str(&format!(
"{SESSION_COOKIE}={token}; Path=/; HttpOnly; SameSite=Lax; Max-Age={}",
days * 86_400
@@ -547,12 +521,13 @@ async fn pick_signal_icon(
if candidates.is_empty() || candidates.len() > 60 {
return Err(ApiError::bad_request("between 1 and 60 icons to choose from"));
}
let icon = crate::signals::pick_icon(&state.ctx, &user.name, &key, &name, &words, description.as_deref(), &candidates)
.await
.map_err(|e| {
tracing::warn!(error = %format!("{e:#}"), "Jev could not pick an icon");
ApiError::bad_gateway("no icon could be picked just now")
})?;
let icon =
crate::signals::pick_icon(&state.ctx, &user.name, &key, &name, &words, description.as_deref(), &candidates)
.await
.map_err(|e| {
tracing::warn!(error = %format!("{e:#}"), "Jev could not pick an icon");
ApiError::bad_gateway("no icon could be picked just now")
})?;
Ok(Json(serde_json::json!({ "icon": icon })))
}
@@ -614,11 +589,7 @@ fn theme_ok(theme: &str, mode: &str) -> bool {
// ---- accounts: admin only ----
fn require_admin(user: &crate::db::User) -> Result<(), ApiError> {
if user.is_admin {
Ok(())
} else {
Err(ApiError::forbidden("only an admin manages accounts"))
}
if user.is_admin { Ok(()) } else { Err(ApiError::forbidden("only an admin manages accounts")) }
}
/// Demoting or removing this account would leave nobody able to manage anyone, and the only
@@ -636,7 +607,8 @@ async fn list_tokens(
let tokens: Vec<_> = state
.ctx
.db
.api_tokens(user.id).await?
.api_tokens(user.id)
.await?
.iter()
.map(|t| serde_json::json!({ "id": t.id, "name": t.name, "created": t.created, "last_used": t.last_used }))
.collect();
@@ -676,15 +648,13 @@ async fn remove_token(
Ok(StatusCode::NO_CONTENT)
}
async fn list_users(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<serde_json::Value>, ApiError> {
async fn list_users(State(state): State<WebState>, user: crate::db::User) -> Result<Json<serde_json::Value>, ApiError> {
require_admin(&user)?;
let users: Vec<_> = state
.ctx
.db
.users().await?
.users()
.await?
.iter()
.map(|u| {
serde_json::json!({
@@ -712,9 +682,8 @@ async fn add_user(
) -> Result<StatusCode, ApiError> {
require_admin(&user)?;
// The same rules as a name a proxy vouches for, so either way of signing in finds it.
let name = crate::auth::name_from_header(&body.name).ok_or_else(|| {
ApiError::bad_request("a name is required, without commas, semicolons or line breaks")
})?;
let name = crate::auth::name_from_header(&body.name)
.ok_or_else(|| ApiError::bad_request("a name is required, without commas, semicolons or line breaks"))?;
if state.ctx.db.user_by_name(&name).await?.is_some() {
return Err(ApiError::bad_request(format!("{name} already exists")));
}
@@ -742,10 +711,8 @@ async fn patch_user(
) -> Result<StatusCode, ApiError> {
require_admin(&user)?;
let users = state.ctx.db.users().await?;
let target = users
.iter()
.find(|u| u.id == id)
.ok_or_else(|| ApiError::bad_request(format!("no account with id {id}")))?;
let target =
users.iter().find(|u| u.id == id).ok_or_else(|| ApiError::bad_request(format!("no account with id {id}")))?;
if !body.admin && last_admin(&users, id) {
return Err(ApiError::bad_request(format!(
"{} is the only admin; make someone else an admin first",
@@ -764,10 +731,8 @@ async fn remove_user(
) -> Result<StatusCode, ApiError> {
require_admin(&user)?;
let users = state.ctx.db.users().await?;
let target = users
.iter()
.find(|u| u.id == id)
.ok_or_else(|| ApiError::bad_request(format!("no account with id {id}")))?;
let target =
users.iter().find(|u| u.id == id).ok_or_else(|| ApiError::bad_request(format!("no account with id {id}")))?;
if last_admin(&users, id) {
return Err(ApiError::bad_request(format!(
"{} is the only admin; make someone else an admin first",
@@ -886,10 +851,7 @@ struct FailingRow {
/// A day, in seconds: how long an error has to hold before the UI mentions it.
const FLAG_AFTER_SECS: i64 = 86_400;
async fn feeds(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<Vec<FeedRow>>, ApiError> {
async fn feeds(State(state): State<WebState>, user: crate::db::User) -> Result<Json<Vec<FeedRow>>, ApiError> {
Ok(Json(feed_rows(&state, &user, None).await?))
}
@@ -928,7 +890,10 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
let crate::db::FeedListing { summary: s, ttl_mins, blocked, unread } = listed.remove(id).unwrap_or_default();
out.push(FeedRow {
private: feed.private,
private_by_address: crate::directory::looks_private(&crate::config::Feed { private: false, ..feed.clone() }),
private_by_address: crate::directory::looks_private(&crate::config::Feed {
private: false,
..feed.clone()
}),
id: id.clone(),
url: feed.url.clone(),
title: s.title,
@@ -952,14 +917,8 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
.or_else(|| feed.group.as_deref().and_then(|g| pauses.get(g)))
.copied()
.unwrap_or(crate::deadair::DEFAULT_PAUSE),
allow_explicit: mine
.allow_explicit
.or(up.and_then(|u| u.allow_explicit))
.unwrap_or(feed.allow_explicit),
auto_download: mine
.auto_download
.or(up.and_then(|u| u.auto_download))
.unwrap_or(feed.auto_download),
allow_explicit: mine.allow_explicit.or(up.and_then(|u| u.allow_explicit)).unwrap_or(feed.allow_explicit),
auto_download: mine.auto_download.or(up.and_then(|u| u.auto_download)).unwrap_or(feed.auto_download),
max_new_per_check: mine
.max_new_per_check
.or(up.and_then(|u| u.max_new_per_check))
@@ -970,10 +929,7 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
// Derived from an OPML and not written to config until you change something.
managed: sub.managed,
schedule: feed.schedule.clone(),
schedule_mins: feed
.schedule
.as_deref()
.and_then(crate::config::parse_interval),
schedule_mins: feed.schedule.as_deref().and_then(crate::config::parse_interval),
every_mins: crate::due_after(&cfg, feed, ttl_mins, None, None, 0) / 60,
last_checked: s.last_checked,
next_check: s
@@ -1037,8 +993,15 @@ fn dir_error(e: crate::directory::DirError) -> ApiError {
/// address, since the directory and this server name a feed differently.
async fn rows_for(state: &WebState, user_id: i64, listed: Vec<ipx_directory::Listing>) -> Result<Vec<PopularRow>> {
// A step each, as the feed list has: a Directory page of 983 ms showed 19 under spans (#250).
let mine: std::collections::HashSet<String> =
state.ctx.db.subscriptions_for(user_id).instrument(step("subscriptions")).await?.into_iter().map(|s| s.feed_id).collect();
let mine: std::collections::HashSet<String> = state
.ctx
.db
.subscriptions_for(user_id)
.instrument(step("subscriptions"))
.await?
.into_iter()
.map(|s| s.feed_id)
.collect();
let here: std::collections::HashMap<String, String> = crate::subscriptions(&state.ctx)
.instrument(step("catalogue"))
.await?
@@ -1066,10 +1029,7 @@ async fn rows_for(state: &WebState, user_id: i64, listed: Vec<ipx_directory::Lis
.collect())
}
async fn get_popular(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<Vec<PopularRow>>, ApiError> {
async fn get_popular(State(state): State<WebState>, user: crate::db::User) -> Result<Json<Vec<PopularRow>>, ApiError> {
let listed = crate::directory::popular(&state.ctx).await.map_err(dir_error)?;
Ok(Json(rows_for(&state, user.id, listed).await?))
}
@@ -1140,14 +1100,22 @@ fn admin_only(user: &crate::db::User) -> Result<(), ApiError> {
if user.is_admin { Ok(()) } else { Err(ApiError::forbidden("only an admin administers the directory")) }
}
async fn remove_listed(State(state): State<WebState>, user: crate::db::User, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
async fn remove_listed(
State(state): State<WebState>,
user: crate::db::User,
Path(id): Path<String>,
) -> Result<StatusCode, ApiError> {
admin_only(&user)?;
crate::directory::remove(&state.ctx, &id).await.map_err(dir_error)?;
tracing::info!(by = %user.name, feed = %id, "removed a feed from the directory");
Ok(StatusCode::NO_CONTENT)
}
async fn refresh_listed(State(state): State<WebState>, user: crate::db::User, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
async fn refresh_listed(
State(state): State<WebState>,
user: crate::db::User,
Path(id): Path<String>,
) -> Result<StatusCode, ApiError> {
admin_only(&user)?;
crate::directory::refresh(&state.ctx, &id).await.map_err(dir_error)?;
Ok(StatusCode::NO_CONTENT)
@@ -1162,14 +1130,23 @@ struct BlockedRow {
at: i64,
}
async fn get_directory_admin(State(state): State<WebState>, user: crate::db::User) -> Result<Json<serde_json::Value>, ApiError> {
async fn get_directory_admin(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<serde_json::Value>, ApiError> {
admin_only(&user)?;
let (instances, blocked) = tokio::join!(crate::directory::instances(&state.ctx), crate::directory::blocked(&state.ctx));
let blocked: Vec<BlockedRow> = blocked.map_err(dir_error)?.into_iter().map(|b| BlockedRow { id: b.id, title: b.title, at: b.at }).collect();
let (instances, blocked) =
tokio::join!(crate::directory::instances(&state.ctx), crate::directory::blocked(&state.ctx));
let blocked: Vec<BlockedRow> =
blocked.map_err(dir_error)?.into_iter().map(|b| BlockedRow { id: b.id, title: b.title, at: b.at }).collect();
Ok(Json(serde_json::json!({ "instances": instances.map_err(dir_error)?, "blocked": blocked })))
}
async fn unblock_listed(State(state): State<WebState>, user: crate::db::User, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
async fn unblock_listed(
State(state): State<WebState>,
user: crate::db::User,
Path(id): Path<String>,
) -> Result<StatusCode, ApiError> {
admin_only(&user)?;
crate::directory::unblock(&state.ctx, &id).await.map_err(dir_error)?;
// Unblocked, it comes back with a snapshot that has it, and this one has not changed.
@@ -1178,7 +1155,11 @@ async fn unblock_listed(State(state): State<WebState>, user: crate::db::User, Pa
Ok(StatusCode::NO_CONTENT)
}
async fn drop_instance(State(state): State<WebState>, user: crate::db::User, Path(name): Path<String>) -> Result<StatusCode, ApiError> {
async fn drop_instance(
State(state): State<WebState>,
user: crate::db::User,
Path(name): Path<String>,
) -> Result<StatusCode, ApiError> {
admin_only(&user)?;
crate::directory::drop_instance(&state.ctx, &name).await.map_err(dir_error)?;
// When it was this server's own, its feeds come back now rather than in half an hour.
@@ -1245,32 +1226,20 @@ impl<E: Into<anyhow::Error>> From<E> for ApiError {
impl ApiError {
fn bad_request(msg: impl Into<String>) -> Self {
Self {
error: anyhow::Error::msg(msg.into()),
status: StatusCode::BAD_REQUEST,
}
Self { error: anyhow::Error::msg(msg.into()), status: StatusCode::BAD_REQUEST }
}
fn forbidden(msg: impl Into<String>) -> Self {
Self {
error: anyhow::Error::msg(msg.into()),
status: StatusCode::FORBIDDEN,
}
Self { error: anyhow::Error::msg(msg.into()), status: StatusCode::FORBIDDEN }
}
fn not_found(msg: impl Into<String>) -> Self {
Self {
error: anyhow::Error::msg(msg.into()),
status: StatusCode::NOT_FOUND,
}
Self { error: anyhow::Error::msg(msg.into()), status: StatusCode::NOT_FOUND }
}
/// A service ipx asks on someone's behalf failed them.
fn bad_gateway(msg: impl Into<String>) -> Self {
Self {
error: anyhow::Error::msg(msg.into()),
status: StatusCode::BAD_GATEWAY,
}
Self { error: anyhow::Error::msg(msg.into()), status: StatusCode::BAD_GATEWAY }
}
}
@@ -1328,7 +1297,8 @@ mod tests {
assert_eq!(signal_icon(Some("brands/apple".into())).ok().flatten().as_deref(), Some("brands/apple"));
assert!(matches!(signal_icon(Some(String::new())), Ok(None)), "empty clears it");
// It becomes part of an address and a style attribute on the page.
for bad in ["solid/../../api/me", "light/robot", "solid/Robot", "solid/x');background:url(x", "robot", "solid/"] {
for bad in ["solid/../../api/me", "light/robot", "solid/Robot", "solid/x');background:url(x", "robot", "solid/"]
{
assert!(signal_icon(Some(bad.into())).is_err(), "{bad}");
}
}
@@ -1341,13 +1311,25 @@ mod tests {
assert!(!theme_ok("modern", "dark\""));
}
#[tokio::test]
async fn the_page_reports_only_what_it_is_meant_to() {
let u = crate::db::User { id: 1, name: "dee".into(), pass_hash: None, is_admin: false, created: None, last_login: None };
let u = crate::db::User {
id: 1,
name: "dee".into(),
pass_hash: None,
is_admin: false,
created: None,
last_login: None,
};
let e = |ev: &str| ClientEvent {
ev: ev.into(), msg: Some("x".repeat(10_000)), src: None, line: None, feed: None, enclosure: None, deadair: None, ms: Some(5),
ev: ev.into(),
msg: Some("x".repeat(10_000)),
src: None,
line: None,
feed: None,
enclosure: None,
deadair: None,
ms: Some(5),
filter: Some("unread".into()),
};
for ev in ["page_error", "rejection", "player_error", "play_start", "stall", "list_shown"] {
@@ -1382,9 +1364,21 @@ mod tests {
fn feed(url: &str) -> crate::config::Feed {
crate::config::Feed {
url: url.into(), folder: None, group: None, media_types: None, schedule: None, keywords: vec![], allow_explicit: false,
auto_download: true, max_new_per_check: None, username: None,
password: None, password_env: None, category: None, listed: false, private: false,
url: url.into(),
folder: None,
group: None,
media_types: None,
schedule: None,
keywords: vec![],
allow_explicit: false,
auto_download: true,
max_new_per_check: None,
username: None,
password: None,
password_env: None,
category: None,
listed: false,
private: false,
}
}
@@ -1396,7 +1390,8 @@ mod tests {
assert!(absent.schedule.is_none() && absent.folder.is_none());
let cleared: FeedPatch =
serde_json::from_str(r#"{"schedule":null,"folder":null,"category":null,"max_new_per_check":null}"#).unwrap();
serde_json::from_str(r#"{"schedule":null,"folder":null,"category":null,"max_new_per_check":null}"#)
.unwrap();
assert_eq!(cleared.schedule, Some(None), "null must mean clear");
assert_eq!(cleared.folder, Some(None));
assert_eq!(cleared.category, Some(None));
@@ -1425,10 +1420,7 @@ mod tests {
assert!(check_url(" ", "a", &feeds).is_err(), "whitespace only");
assert!(check_url("not a url", "a", &feeds).is_err(), "unparseable");
assert!(check_url("file:///etc/passwd", "a", &feeds).is_err(), "not http(s)");
assert!(
check_url("https://b.example/rss", "a", &feeds).is_err(),
"another feed already has that URL"
);
assert!(check_url("https://b.example/rss", "a", &feeds).is_err(), "another feed already has that URL");
}
}
@@ -1502,11 +1494,20 @@ async fn entry_page(
let filter = crate::db::Filter::parse(page.filter.as_deref().unwrap_or("all"));
let search = page.q.as_deref().map(str::trim).filter(|q| !q.is_empty());
let db = &state.ctx.db;
let order = crate::db::order_sql(page.sort.as_deref().unwrap_or("published"), page.dir.as_deref().unwrap_or("desc"));
let order =
crate::db::order_sql(page.sort.as_deref().unwrap_or("published"), page.dir.as_deref().unwrap_or("desc"));
let limit = page.limit.clamp(1, 200);
let mut rows = match signal {
Some(id) => db.signal_entries(user_id, id, filter, search, page.offset, limit, &order).instrument(step("entries_in")).await?,
None => db.entries_in(user_id, feed, filter, search, page.offset, limit, &order).instrument(step("entries_in")).await?,
Some(id) => {
db.signal_entries(user_id, id, filter, search, page.offset, limit, &order)
.instrument(step("entries_in"))
.await?
}
None => {
db.entries_in(user_id, feed, filter, search, page.offset, limit, &order)
.instrument(step("entries_in"))
.await?
}
};
step("sanitize").in_scope(|| {
let mut sanitizer = feed_sanitizer();
@@ -1737,8 +1738,11 @@ async fn patch_feed(
// The rest describes the feed itself -- where its files land, its address, when it is
// polled -- and there is one of those however many people read it.
let admin_only =
body.url.is_some() || body.folder.is_some() || body.schedule.is_some() || body.category.is_some() || body.private == Some(false);
let admin_only = body.url.is_some()
|| body.folder.is_some()
|| body.schedule.is_some()
|| body.category.is_some()
|| body.private == Some(false);
if !admin_only && body.private.is_none() {
return Ok(StatusCode::NO_CONTENT);
}
@@ -1753,10 +1757,8 @@ async fn patch_feed(
// entry: promote it, so the config holds your decisions and nothing else.
if !cfg.feeds.contains_key(&id) {
let subs = crate::subscriptions(&state.ctx).await?;
let found = subs
.iter()
.find(|s| s.id == id)
.ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?;
let found =
subs.iter().find(|s| s.id == id).ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?;
cfg.feeds.insert(id.clone(), found.cfg.clone());
state.ctx.db.unmanage(&id).await?;
}
@@ -1766,10 +1768,7 @@ async fn patch_feed(
None => None,
};
let feed = cfg
.feeds
.get_mut(&id)
.ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?;
let feed = cfg.feeds.get_mut(&id).ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?;
let mut url_changed = false;
if let Some(url) = checked {
@@ -1847,11 +1846,7 @@ async fn download_now(
Path(id): Path<i64>,
user: crate::db::User,
) -> Result<StatusCode, ApiError> {
let enc = state
.ctx
.db
.enclosure(id).await?
.ok_or_else(|| ApiError::not_found(format!("no file {id}")))?;
let enc = state.ctx.db.enclosure(id).await?.ok_or_else(|| ApiError::not_found(format!("no file {id}")))?;
subscribed(&state, &user, &enc.feed_id).await?;
if enc.path.is_some() {
return Ok(StatusCode::NO_CONTENT); // Already here.
@@ -1877,11 +1872,7 @@ async fn delete_file(
user: crate::db::User,
Query(q): Query<Force>,
) -> Result<StatusCode, ApiError> {
let enc = state
.ctx
.db
.enclosure(id).await?
.ok_or_else(|| ApiError::not_found(format!("no file {id}")))?;
let enc = state.ctx.db.enclosure(id).await?.ok_or_else(|| ApiError::not_found(format!("no file {id}")))?;
subscribed(&state, &user, &enc.feed_id).await?;
// There is one copy of the file: deleting it deletes everyone's. Say so before doing
@@ -1891,7 +1882,11 @@ async fn delete_file(
let people = |n: i64| if n == 1 { "person".to_string() } else { format!("{n} people") };
let complaint = match (starred, unread) {
(0, 0) => None,
(0, u) => Some(format!("{} subscribed to this feed {} not played it yet", people(u), if u == 1 { "has" } else { "have" })),
(0, u) => Some(format!(
"{} subscribed to this feed {} not played it yet",
people(u),
if u == 1 { "has" } else { "have" }
)),
(st, 0) => Some(format!("another {} pinned it", people(st))),
(st, u) => Some(format!(
"another {} pinned it, and {} not played it yet",
@@ -1914,10 +1909,10 @@ async fn delete_file(
}
// The row survives as 'reaped', which is what stops the next scan re-downloading it.
state.ctx.db.mark_reaped(id).await?;
state.events.send(Event::Reaped {
path: enc.path.unwrap_or_default(),
bytes: enc.length.unwrap_or(0).max(0) as u64,
}).ok();
state
.events
.send(Event::Reaped { path: enc.path.unwrap_or_default(), bytes: enc.length.unwrap_or(0).max(0) as u64 })
.ok();
Ok(StatusCode::NO_CONTENT)
}
@@ -2000,12 +1995,7 @@ fn changed_feed(ev: &Event) -> Option<&str> {
}
/// Audio, served by ServeFile so Range requests work and the player can seek.
async fn media(
State(state): State<WebState>,
Path(id): Path<i64>,
user: crate::db::User,
req: Request,
) -> Response {
async fn media(State(state): State<WebState>, Path(id): Path<i64>, user: crate::db::User, req: Request) -> Response {
serve_media(state, id, user, req, false).await
}
@@ -2171,7 +2161,9 @@ async fn art(State(state): State<WebState>, Query(q): Query<ArtQuery>) -> Respon
/// Spotify's CloudFront `application/octet-stream`, pippa.io none at all (#279).
fn untyped(said: &str) -> bool {
let said = said.split(';').next().unwrap_or("").trim();
said.is_empty() || said.eq_ignore_ascii_case("application/octet-stream") || said.eq_ignore_ascii_case("binary/octet-stream")
said.is_empty()
|| said.eq_ignore_ascii_case("application/octet-stream")
|| said.eq_ignore_ascii_case("binary/octet-stream")
}
/// The artwork's type: the publisher's when it says image, else what the first bytes show, as a
@@ -2235,7 +2227,8 @@ async fn read_all(
// A subscription's own row has no entries, so marking it read means everything under it.
let mut ids = vec![id.clone()];
ids.extend(
crate::subscriptions(&state.ctx).await?
crate::subscriptions(&state.ctx)
.await?
.into_iter()
.filter(|s| s.cfg.group.as_deref() == Some(id.as_str()))
.map(|s| s.id),
@@ -2250,8 +2243,7 @@ async fn read_all_mine(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Json<serde_json::Value>, ApiError> {
let ids: Vec<String> =
state.ctx.db.subscriptions_for(user.id).await?.into_iter().map(|s| s.feed_id).collect();
let ids: Vec<String> = state.ctx.db.subscriptions_for(user.id).await?.into_iter().map(|s| s.feed_id).collect();
let n = state.ctx.db.mark_all_read(user.id, &ids).await?;
Ok(Json(serde_json::json!({ "marked": n })))
}
@@ -2288,19 +2280,13 @@ async fn download_latest(
}
/// Subscriptions as OPML, so they can move to another podcast app.
async fn export_opml(
State(state): State<WebState>,
user: crate::db::User,
) -> Result<Response, ApiError> {
async fn export_opml(State(state): State<WebState>, user: crate::db::User) -> Result<Response, ApiError> {
// Yours, not the whole catalogue: other people's feeds, and any private URLs in them, are
// not yours to download. This used to export config.toml to whoever asked.
let mine: std::collections::HashSet<String> =
state.ctx.db.subscriptions_for(user.id).await?.into_iter().map(|s| s.feed_id).collect();
let mut doc = opml::OPML {
head: Some(opml::Head {
title: Some("ipx subscriptions".into()),
..Default::default()
}),
head: Some(opml::Head { title: Some("ipx subscriptions".into()), ..Default::default() }),
..Default::default()
};
for s in crate::subscriptions(&state.ctx).await? {
@@ -2308,23 +2294,15 @@ async fn export_opml(
if s.managed || !mine.contains(&s.id) {
continue;
}
let title = state
.ctx
.db
.feed_summary(&s.id).await
.ok()
.and_then(|sum| sum.title)
.unwrap_or_else(|| s.id.clone());
let title =
state.ctx.db.feed_summary(&s.id).await.ok().and_then(|sum| sum.title).unwrap_or_else(|| s.id.clone());
doc.add_feed(&title, &s.cfg.url);
}
let xml = doc.to_string().map_err(|e| anyhow::anyhow!("writing OPML: {e}"))?;
Ok((
[
(header::CONTENT_TYPE, "text/x-opml; charset=utf-8"),
(
header::CONTENT_DISPOSITION,
"attachment; filename=\"ipx-subscriptions.opml\"",
),
(header::CONTENT_DISPOSITION, "attachment; filename=\"ipx-subscriptions.opml\""),
],
xml,
)
@@ -2343,8 +2321,8 @@ async fn import_opml(
) -> Result<Json<serde_json::Value>, ApiError> {
// Refused before anything is touched. Nothing reaches the disk either way: an uploaded
// file arrives as text, is read here, and is gone when the request ends.
let doc = opml::OPML::from_str(&body.xml)
.map_err(|e| ApiError::bad_request(format!("that is not an OPML file: {e}")))?;
let doc =
opml::OPML::from_str(&body.xml).map_err(|e| ApiError::bad_request(format!("that is not an OPML file: {e}")))?;
let (added, already) = crate::subscribe_opml(&state.ctx, &doc, user.id).await?;
if added > 0 {
scan_soon(&state, None).await;
@@ -2448,11 +2426,7 @@ async fn patch_settings(
cfg.general.max_new_per_check = v;
}
if let Some(v) = body.media_types {
cfg.general.media_types = v
.into_iter()
.map(|t| t.trim().to_lowercase())
.filter(|t| !t.is_empty())
.collect();
cfg.general.media_types = v.into_iter().map(|t| t.trim().to_lowercase()).filter(|t| !t.is_empty()).collect();
}
if let Some(v) = body.max_total_gb {
cfg.general.max_total_gb = v.max(0.0);
@@ -2466,7 +2440,9 @@ async fn patch_settings(
if let Some(v) = body.directory_url {
let v = v.trim().trim_end_matches('/').to_owned();
if !v.is_empty() && !url::Url::parse(&v).is_ok_and(|u| matches!(u.scheme(), "http" | "https")) {
return Err(ApiError::bad_request(format!("{v:?} is not a web address; try https://directory.example.net")));
return Err(ApiError::bad_request(format!(
"{v:?} is not a web address; try https://directory.example.net"
)));
}
cfg.general.directory_url = v;
}
@@ -2568,7 +2544,8 @@ async fn client_event(user: crate::db::User, Json(e): Json<ClientEvent>) -> Stat
// From the browser: only the kinds the page sends, and nothing long enough to fill the log.
let cut = |s: &Option<String>| s.as_deref().map(|s| s.chars().take(500).collect::<String>());
let (msg, src, feed, filter) = (cut(&e.msg), cut(&e.src), cut(&e.feed), cut(&e.filter));
let (ev, line, enclosure, deadair, ms, user) = (e.ev.as_str(), e.line, e.enclosure, e.deadair, e.ms, user.name.as_str());
let (ev, line, enclosure, deadair, ms, user) =
(e.ev.as_str(), e.line, e.enclosure, e.deadair, e.ms, user.name.as_str());
match ev {
"page_error" | "rejection" | "player_error" => tracing::warn!(
target: "ipx::client", ev, msg, src, line, feed, enclosure, deadair, user,
@@ -2588,7 +2565,6 @@ async fn client_event(user: crate::db::User, Json(e): Json<ClientEvent>) -> Stat
StatusCode::NO_CONTENT
}
/// Names a request's trace by its route, `POST /api/entries/{feed_id}/{guid}/flags`, once routing
/// has found it. Named by the path `access_log` sees, every item's GUID was a trace name of its
/// own, and nothing grouped. A route layer, because only one runs after routing. Renamed on the

4
rustfmt.toml Normal file
View File

@@ -0,0 +1,4 @@
# Matches how the code was written by hand before rustfmt was adopted: long lines, and short
# expressions kept on one line.
max_width = 120
use_small_heuristics = "Max"