rays/ipx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity

Format the Rust code with rustfmt

Browse Source 
The code was formatted by hand, and `cargo fmt --check` differed in 866
places. rustfmt.toml keeps the house's long lines (120) and short
expressions on one line, which halves what changes; this commit is only
what rustfmt does with it, and is named in .git-blame-ignore-revs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-10-17 21:55:41 +00:00
parent 2c4ba7896e
commit 692a510217
26 changed files with 1710 additions and 1114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
crates/directory/src/apple.rs
View File

@@ -96,8 +96,11 @@ async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<(
}
let result = dir.store.snapshot(INSTANCE, &entries).await?;
tracing::info!(
shows = ids.len(), accepted = result.accepted, refused_private = result.refused_private,
refused_blocked = result.refused_blocked, refused_invalid = result.refused_invalid,
shows = ids.len(),
accepted = result.accepted,
refused_private = result.refused_private,
refused_blocked = result.refused_blocked,
refused_invalid = result.refused_invalid,
"Apple Podcasts synced"
);
// Fetched now, not when the refresher next looks.
@@ -128,7 +131,11 @@ struct Genre {
fn categories(g: &Genres) -> BTreeMap<String, Vec<String>> {
g.0.get("26")
.map(|podcasts| {
podcasts.subgenres.values().map(|c| (c.name.clone(), c.subgenres.values().map(|s| s.name.clone()).collect())).collect()
podcasts
.subgenres
.values()
.map(|c| (c.name.clone(), c.subgenres.values().map(|s| s.name.clone()).collect()))
.collect()
})
.unwrap_or_default()
}
@@ -253,7 +260,8 @@ mod tests {
)
.unwrap();
assert_eq!(chart_ids(&chart), ["1200361736", "1028908750"]);
let one: Chart = serde_json::from_str(r#"{"feed": {"entry": {"id": {"attributes": {"im:id": "7"}}}}}"#).unwrap();
let one: Chart =
serde_json::from_str(r#"{"feed": {"entry": {"id": {"attributes": {"im:id": "7"}}}}}"#).unwrap();
assert_eq!(chart_ids(&one), ["7"], "a chart of one");
let empty: Chart = serde_json::from_str(r#"{"feed": {}}"#).unwrap();
assert!(chart_ids(&empty).is_empty());

9
crates/directory/src/config.rs
View File

@@ -162,7 +162,10 @@ mod tests {
let cfg = Config::parse(GOOD).unwrap();
assert_eq!(cfg.bind, "0.0.0.0:8100");
let t = cfg.tokens().unwrap();
assert_eq!((t[0].name.as_str(), t[0].access, t[0].sha256.as_str()), ("tower", Access::Rwa, "a".repeat(64).as_str()));
assert_eq!(
(t[0].name.as_str(), t[0].access, t[0].sha256.as_str()),
("tower", Access::Rwa, "a".repeat(64).as_str())
);
assert_eq!(t[1].access, Access::Ro);
assert!(Access::Ro < Access::Rw && Access::Rw < Access::Rwa);
}
@@ -171,7 +174,9 @@ mod tests {
fn a_missing_or_misspelled_access_a_reused_name_or_a_bad_hash_is_refused() {
let h = "a".repeat(64);
assert!(Config::parse(&format!("[[tokens]]\nname = \"a\"\ntoken_sha256 = \"{h}\"")).is_err());
assert!(Config::parse(&format!("[[tokens]]\nname = \"a\"\naccess = \"admin\"\ntoken_sha256 = \"{h}\"")).is_err());
assert!(
Config::parse(&format!("[[tokens]]\nname = \"a\"\naccess = \"admin\"\ntoken_sha256 = \"{h}\"")).is_err()
);
assert!(Config::parse(&GOOD.replace("\"friend\"", "\"tower\"")).is_err());
assert!(Config::parse(&GOOD.replace(&"b".repeat(64), "xyz")).unwrap().tokens().is_err());
assert!(Config::parse(&GOOD.replace(&"b".repeat(64), &h)).unwrap().tokens().is_err());

83
crates/directory/src/lib.rs
View File

@@ -153,16 +153,16 @@ impl Directory {
fn caller(&self, presented: &str) -> Option<Me> {
let h = config::hash(presented);
// Every token compared, whether or not one matched, so timing says nothing either way.
self.tokens
.read()
.unwrap()
.iter()
.fold(None, |found, t| if config::same_secret(&t.sha256, &h) { Some(Me { name: t.name.clone(), access: t.access }) } else { found })
self.tokens.read().unwrap().iter().fold(None, |found, t| {
if config::same_secret(&t.sha256, &h) { Some(Me { name: t.name.clone(), access: t.access }) } else { found }
})
}
/// Fetches one feed and stores what it found.
pub async fn refresh(&self, due: &store::Due) -> anyhow::Result<()> {
let got = match ipx_feed::fetch(&self.client, &due.url, None, due.etag.as_deref(), due.last_modified.as_deref()).await {
let got = match ipx_feed::fetch(&self.client, &due.url, None, due.etag.as_deref(), due.last_modified.as_deref())
.await
{
Ok((ipx_feed::Fetched::NotModified, _)) => Refreshed::NotModified,
Ok((ipx_feed::Fetched::Body { bytes, etag, last_modified }, _)) => {
if ipx_feed::is_opml(&bytes) {
@@ -242,10 +242,14 @@ pub async fn reloader(dir: Shared, path: std::path::PathBuf) {
let n = tokens.len();
match dir.set_tokens(tokens).await {
Ok(dropped) => tracing::info!(tokens = n, dropped, "config read again"),
Err(e) => tracing::warn!(error = %format!("{e:#}"), "config read again, but old snapshots could not be dropped"),
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "config read again, but old snapshots could not be dropped")
}
}
}
Err(e) => tracing::warn!(error = %format!("{e:#}"), "config changed but does not load; keeping the tokens as they were"),
Err(e) => {
tracing::warn!(error = %format!("{e:#}"), "config changed but does not load; keeping the tokens as they were")
}
}
}
}
@@ -342,7 +346,8 @@ async fn authenticate(State(dir): State<Shared>, mut req: Request, next: Next) -
.map(str::trim)
.filter(|t| !t.is_empty());
let Some(me) = presented.and_then(|t| dir.caller(t)) else {
return ApiError(StatusCode::UNAUTHORIZED, "a valid token is needed: Authorization: Bearer <token>".into()).into_response();
return ApiError(StatusCode::UNAUTHORIZED, "a valid token is needed: Authorization: Bearer <token>".into())
.into_response();
};
let who = Who(me.name.clone());
req.extensions_mut().insert(me);
@@ -387,7 +392,11 @@ async fn popular(State(dir): State<Shared>, Query(q): Query<Limit>) -> Result<Js
}
async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
dir.store.feed(&id).await?.map(Json).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))
dir.store
.feed(&id)
.await?
.map(Json)
.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))
}
async fn snapshot(
@@ -408,7 +417,11 @@ async fn snapshot(
Ok(Json(r))
}
async fn remove(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
async fn remove(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(id): Path<String>,
) -> Result<StatusCode, ApiError> {
need(&me, Access::Rwa)?;
dir.store.block(&id).await?;
tracing::info!(feed = %id, by = %me.name, "feed removed and blocked");
@@ -420,7 +433,11 @@ async fn blocked(State(dir): State<Shared>, Extension(me): Extension<Me>) -> Res
Ok(Json(dir.store.blocked().await?))
}
async fn unblock(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
async fn unblock(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(id): Path<String>,
) -> Result<StatusCode, ApiError> {
need(&me, Access::Rwa)?;
if !dir.store.unblock(&id).await? {
return Err(ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not blocked")));
@@ -434,16 +451,28 @@ async fn instances(State(dir): State<Shared>, Extension(me): Extension<Me>) -> R
Ok(Json(dir.store.instances().await?))
}
async fn drop_instance(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(name): Path<String>) -> Result<StatusCode, ApiError> {
async fn drop_instance(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(name): Path<String>,
) -> Result<StatusCode, ApiError> {
need(&me, Access::Rwa)?;
let n = dir.store.drop_instance(&name).await?;
tracing::info!(instance = %name, feeds = n, by = %me.name, "snapshot dropped");
Ok(StatusCode::NO_CONTENT)
}
async fn refresh(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
async fn refresh(
State(dir): State<Shared>,
Extension(me): Extension<Me>,
Path(id): Path<String>,
) -> Result<Json<FeedPage>, ApiError> {
need(&me, Access::Rwa)?;
let due = dir.store.due_one(&id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))?;
let due = dir
.store
.due_one(&id)
.await?
.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))?;
dir.refresh(&due).await?;
feed(State(dir), Path(id)).await
}
@@ -467,7 +496,13 @@ mod tests {
(router(dir.clone()), dir)
}
async fn call(app: &Router, method: &str, path: &str, token: Option<&str>, body: Option<serde_json::Value>) -> (u16, serde_json::Value) {
async fn call(
app: &Router,
method: &str,
path: &str,
token: Option<&str>,
body: Option<serde_json::Value>,
) -> (u16, serde_json::Value) {
let mut req = axum::http::Request::builder().method(method).uri(path);
if let Some(t) = token {
req = req.header("authorization", format!("Bearer secret-{t}"));
@@ -501,7 +536,8 @@ mod tests {
.unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), 200);
let got: serde_json::Value = serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap();
let got: serde_json::Value =
serde_json::from_slice(&axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap()).unwrap();
assert_eq!(got["accepted"], 1);
}
@@ -510,11 +546,8 @@ mod tests {
let (app, _) = app().await;
for (asked, sent) in [("br", Some("br")), ("zstd", Some("zstd")), ("gzip", Some("gzip")), ("identity", None)] {
// A page, not an empty feed list: under 32 bytes is sent as it is.
let req = axum::http::Request::builder()
.uri("/")
.header("accept-encoding", asked)
.body(Body::empty())
.unwrap();
let req =
axum::http::Request::builder().uri("/").header("accept-encoding", asked).body(Body::empty()).unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), 200);
let got = resp.headers().get("content-encoding").map(|v| v.to_str().unwrap().to_owned());
@@ -583,7 +616,11 @@ mod tests {
assert_eq!(call(&app, "GET", "/v1/feeds/nope", Some("ro"), None).await.0, 404);
assert_eq!(call(&app, "GET", "/v1/popular?limit=5", Some("ro"), None).await.1[0]["id"], id.as_str());
for (m, p) in [("DELETE", format!("/v1/feeds/{id}")), ("GET", "/v1/blocked".into()), ("DELETE", "/v1/instances/rw-token".into())] {
for (m, p) in [
("DELETE", format!("/v1/feeds/{id}")),
("GET", "/v1/blocked".into()),
("DELETE", "/v1/instances/rw-token".into()),
] {
assert_eq!(call(&app, m, &p, Some("rw"), None).await.0, 403, "{m} {p}");
}
assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), Some("rwa"), None).await.0, 204);

25
crates/directory/src/main.rs
View File

@@ -74,15 +74,24 @@ async fn main() -> Result<()> {
async fn serve(path: &std::path::Path) -> Result<()> {
// One JSON object a line for Loki when DIRECTORY_LOG_FORMAT=json, as ipx does; text otherwise.
let json = std::env::var("DIRECTORY_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
let filter = || tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
let filter =
|| tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
// Traces to Tempo when OTEL_EXPORTER_OTLP_ENDPOINT is set, joining ipx's (#213).
let otel = ipx_directory::otel::provider("ipx-directory")?;
tracing_subscriber::registry()
.with((!json).then(|| tracing_subscriber::fmt::layer().with_writer(std::io::stderr).with_filter(filter())))
.with(json.then(|| tracing_subscriber::fmt::layer().json().flatten_event(true).with_writer(std::io::stderr).with_filter(filter())))
.with(json.then(|| {
tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_writer(std::io::stderr)
.with_filter(filter())
}))
.with(otel.as_ref().map(|p| {
use opentelemetry::trace::TracerProvider;
tracing_opentelemetry::layer().with_tracer(p.tracer("ipx-directory")).with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info,ipx_feed=info"))
tracing_opentelemetry::layer()
.with_tracer(p.tracer("ipx-directory"))
.with_filter(tracing_subscriber::EnvFilter::new("ipx_directory=info,ipx_feed=info"))
}))
.init();
@@ -96,10 +105,13 @@ async fn serve(path: &std::path::Path) -> Result<()> {
let dir = Directory::new(store, vec![])?;
let dropped = dir.set_tokens(tokens).await?;
let listener = tokio::net::TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
let listener =
tokio::net::TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
tracing::info!(version = env!("CARGO_PKG_VERSION"), bind = %cfg.bind, tokens = cfg.tokens.len(), dropped, "directory started");
if !cfg.metrics_bind.is_empty() {
let metrics = tokio::net::TcpListener::bind(&cfg.metrics_bind).await.with_context(|| format!("listening for metrics on {}", cfg.metrics_bind))?;
let metrics = tokio::net::TcpListener::bind(&cfg.metrics_bind)
.await
.with_context(|| format!("listening for metrics on {}", cfg.metrics_bind))?;
tracing::info!(bind = %cfg.metrics_bind, "serving metrics for Prometheus at /metrics");
tokio::spawn(ipx_directory::metrics::serve(metrics));
}
@@ -112,7 +124,8 @@ async fn serve(path: &std::path::Path) -> Result<()> {
tokio::spawn(ipx_directory::reloader(dir.clone(), path.to_owned()));
axum::serve(listener, ipx_directory::router(dir))
.with_graceful_shutdown(async {
let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("SIGTERM handler");
let mut term =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("SIGTERM handler");
tokio::select! {
_ = term.recv() => {}
_ = tokio::signal::ctrl_c() => {}

57
crates/directory/src/metrics.rs
View File

@@ -14,7 +14,8 @@ static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
/// The process's resident memory in bytes and its open files, from /proc; None elsewhere.
pub fn process() -> (Option<f64>, Option<f64>) {
let rss = std::fs::read_to_string("/proc/self/status").ok().and_then(|s| {
let kb = s.lines().find_map(|l| l.strip_prefix("VmRSS:"))?.trim().strip_suffix("kB")?.trim().parse::<f64>().ok()?;
let kb =
s.lines().find_map(|l| l.strip_prefix("VmRSS:"))?.trim().strip_suffix("kB")?.trim().parse::<f64>().ok()?;
Some(kb * 1024.0)
});
let fds = std::fs::read_dir("/proc/self/fd").ok().map(|d| d.count() as f64);
@@ -35,10 +36,34 @@ fn render(started: std::time::Instant) -> String {
if let Some(v) = fds {
gauge("directory_process_open_fds", "gauge", "Files and sockets the process has open.", "", v);
}
gauge("directory_feed_fetches_total", "counter", "Feeds fetched, answered 304 included.", "", ipx_feed::FETCHES.load(Relaxed) as f64);
gauge("directory_feed_bytes_total", "counter", "Bytes of feed bodies fetched.", "", ipx_feed::FETCHED_BYTES.load(Relaxed) as f64);
gauge("directory_uptime_seconds", "gauge", "Seconds since the directory started.", "", started.elapsed().as_secs() as f64);
gauge("directory_build_info", "gauge", "The version running, as a label.", &format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")), 1.0);
gauge(
"directory_feed_fetches_total",
"counter",
"Feeds fetched, answered 304 included.",
"",
ipx_feed::FETCHES.load(Relaxed) as f64,
);
gauge(
"directory_feed_bytes_total",
"counter",
"Bytes of feed bodies fetched.",
"",
ipx_feed::FETCHED_BYTES.load(Relaxed) as f64,
);
gauge(
"directory_uptime_seconds",
"gauge",
"Seconds since the directory started.",
"",
started.elapsed().as_secs() as f64,
);
gauge(
"directory_build_info",
"gauge",
"The version running, as a label.",
&format!("{{version=\"{}\"}}", env!("CARGO_PKG_VERSION")),
1.0,
);
out
}
@@ -46,14 +71,15 @@ fn render(started: std::time::Instant) -> String {
/// Access in front, and these are for the LAN.
pub async fn serve(listener: tokio::net::TcpListener) {
let started = std::time::Instant::now();
let app = axum::Router::new().route(
"/metrics",
axum::routing::get(move || async move {
([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
}),
)
// Prometheus asks for gzip (#253).
.layer(tower_http::compression::CompressionLayer::new().gzip(true));
let app = axum::Router::new()
.route(
"/metrics",
axum::routing::get(move || async move {
([(axum::http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], render(started))
}),
)
// Prometheus asks for gzip (#253).
.layer(tower_http::compression::CompressionLayer::new().gzip(true));
if let Err(e) = axum::serve(listener, app).await {
tracing::error!(error = %e, "metrics stopped");
}
@@ -64,6 +90,9 @@ mod tests {
#[test]
fn metrics_say_memory_and_feeds() {
let text = super::render(std::time::Instant::now());
assert!(text.contains("directory_process_resident_bytes ") && text.contains("directory_feed_bytes_total "), "{text}");
assert!(
text.contains("directory_process_resident_bytes ") && text.contains("directory_feed_bytes_total "),
"{text}"
);
}
}

14
crates/directory/src/otel.rs
View File

@@ -70,13 +70,21 @@ mod tests {
assert_eq!(cx.span().span_context().trace_id().to_string(), "4bf92f3577b34da6a3ce929d0e0e4736");
assert!(cx.span().span_context().is_remote());
// Anything else from the browser starts a trace of its own.
for bad in ["", "00-xyz-00f067aa0ba902b7-01", "01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
"00-00000000000000000000000000000000-00f067aa0ba902b7-01", "00-4bf92f3577b34da6a3ce929d0e0e4736"] {
for bad in [
"",
"00-xyz-00f067aa0ba902b7-01",
"01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
"00-00000000000000000000000000000000-00f067aa0ba902b7-01",
"00-4bf92f3577b34da6a3ce929d0e0e4736",
] {
assert!(with(bad).is_none(), "{bad}");
}
assert!(remote_parent(&axum::http::HeaderMap::new(), None).is_none());
// From the query, where a beacon or Safari's player cannot send a header (#192).
let q = remote_parent(&axum::http::HeaderMap::new(), Some("x=1&traceparent=00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"));
let q = remote_parent(
&axum::http::HeaderMap::new(),
Some("x=1&traceparent=00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
);
assert_eq!(q.unwrap().span().span_context().trace_id().to_string(), "4bf92f3577b34da6a3ce929d0e0e4736");
assert!(remote_parent(&axum::http::HeaderMap::new(), Some("traceparent=nonsense")).is_none());
}

134
crates/directory/src/site.rs
View File

@@ -9,11 +9,11 @@
use std::collections::BTreeMap;
use std::fmt::Write;
use axum::Router;
use axum::extract::{Path, Query, State};
use axum::http::{HeaderValue, StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::Router;
use crate::{Listing, Shared};
@@ -63,7 +63,9 @@ async fn home(State(dir): State<Shared>) -> Response {
Err(e) => return failed(e),
};
let mut top = all.clone();
top.sort_by(|a, b| b.subscribers.cmp(&a.subscribers).then_with(|| name(a).to_lowercase().cmp(&name(b).to_lowercase())));
top.sort_by(|a, b| {
b.subscribers.cmp(&a.subscribers).then_with(|| name(a).to_lowercase().cmp(&name(b).to_lowercase()))
});
top.truncate(10);
let mut cats: BTreeMap<&str, Vec<&Listing>> = BTreeMap::new();
for l in &all {
@@ -75,12 +77,25 @@ async fn home(State(dir): State<Shared>) -> Response {
} else {
b.push_str("<h2>Most subscribed</h2><ol class=chart>");
for (i, l) in top.iter().enumerate() {
let _ = write!(b, "<li><a href=\"/feed/{}\"><span class=no>{}</span>{}<span class=txt><b>{}</b><small>{}</small></span></a></li>",
esc(&l.id), i + 1, art(l, "art sm"), esc(name(l)), subscribers(l.subscribers));
let _ = write!(
b,
"<li><a href=\"/feed/{}\"><span class=no>{}</span>{}<span class=txt><b>{}</b><small>{}</small></span></a></li>",
esc(&l.id),
i + 1,
art(l, "art sm"),
esc(name(l)),
subscribers(l.subscribers)
);
}
b.push_str("</ol><h2>Categories</h2><div class=cats>");
for (c, ls) in &cats {
let _ = write!(b, "<a class=cat href=\"/category/{}\"><b>{}</b><small>{}</small></a>", seg(c), esc(c), ls.len());
let _ = write!(
b,
"<a class=cat href=\"/category/{}\"><b>{}</b><small>{}</small></a>",
seg(c),
esc(c),
ls.len()
);
}
let _ = write!(b, "</div><p><a class=more href=\"/all\">All {} feeds, A to Z</a></p>", all.len());
}
@@ -115,10 +130,11 @@ async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn
if ls.is_empty() && here.starts_with("/category/") {
return not_found("That category has nothing in it.");
}
let subs: BTreeMap<String, usize> = ls.iter().filter_map(|l| l.subcategory.clone()).fold(BTreeMap::new(), |mut m, s| {
*m.entry(s).or_default() += 1;
m
});
let subs: BTreeMap<String, usize> =
ls.iter().filter_map(|l| l.subcategory.clone()).fold(BTreeMap::new(), |mut m, s| {
*m.entry(s).or_default() += 1;
m
});
ls.retain(|l| f.keep(l) && f.sub.as_ref().is_none_or(|s| l.subcategory.as_ref() == Some(s)));
ls.sort_by_key(|l| name(l).to_lowercase());
@@ -138,18 +154,38 @@ async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn
if q.is_empty() { here.to_owned() } else { format!("{here}?{q}") }
};
let mut b = String::new();
let _ = write!(b, "<p class=back><a href=\"/\">Directory</a></p><h2>{}</h2><nav class=tabs aria-label=Kind>", esc(title));
let _ = write!(
b,
"<p class=back><a href=\"/\">Directory</a></p><h2>{}</h2><nav class=tabs aria-label=Kind>",
esc(title)
);
for (k, label) in [(None, "All"), (Some("podcasts"), "Podcasts"), (Some("blogs"), "Blogs")] {
let on = f.kind.as_deref() == k;
let _ = write!(b, "<a href=\"{}\"{}>{label}</a>", esc(&link(k, f.sub.as_deref())), if on { " aria-current=page" } else { "" });
let _ = write!(
b,
"<a href=\"{}\"{}>{label}</a>",
esc(&link(k, f.sub.as_deref())),
if on { " aria-current=page" } else { "" }
);
}
b.push_str("</nav>");
if !subs.is_empty() {
b.push_str("<nav class=pills aria-label=Subcategory>");
let _ = write!(b, "<a href=\"{}\"{}>All</a>", esc(&link(f.kind.as_deref(), None)), if f.sub.is_none() { " aria-current=page" } else { "" });
let _ = write!(
b,
"<a href=\"{}\"{}>All</a>",
esc(&link(f.kind.as_deref(), None)),
if f.sub.is_none() { " aria-current=page" } else { "" }
);
for (s, n) in &subs {
let on = f.sub.as_deref() == Some(s.as_str());
let _ = write!(b, "<a href=\"{}\"{}>{} <small>{n}</small></a>", esc(&link(f.kind.as_deref(), Some(s))), if on { " aria-current=page" } else { "" }, esc(s));
let _ = write!(
b,
"<a href=\"{}\"{}>{} <small>{n}</small></a>",
esc(&link(f.kind.as_deref(), Some(s))),
if on { " aria-current=page" } else { "" },
esc(s)
);
}
b.push_str("</nav>");
}
@@ -158,7 +194,14 @@ async fn listed(dir: &Shared, title: &str, here: &str, f: &Filter, pick: impl Fn
} else {
let _ = write!(b, "<p class=count>{}</p><div class=tiles>", plural(ls.len(), "feed"));
for l in &ls {
let _ = write!(b, "<a class=tile href=\"/feed/{}\">{}<b>{}</b><small>{}</small></a>", esc(&l.id), art(l, "art"), esc(name(l)), subscribers(l.subscribers));
let _ = write!(
b,
"<a class=tile href=\"/feed/{}\">{}<b>{}</b><small>{}</small></a>",
esc(&l.id),
art(l, "art"),
esc(name(l)),
subscribers(l.subscribers)
);
}
b.push_str("</div>");
}
@@ -177,11 +220,21 @@ async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Response {
if let Some(c) = &l.category {
let _ = write!(b, " <span aria-hidden=true>/</span> <a href=\"/category/{}\">{}</a>", seg(c), esc(c));
}
let _ = write!(b, "</p><div class=show>{}<div class=meta><h2>{}</h2><p class=sub>{}{}</p>", art(l, "art lg"), esc(name(l)),
subscribers(l.subscribers), l.subcategory.as_deref().map(|s| format!(" &middot; {}", esc(s))).unwrap_or_default());
let _ = write!(b, "<div class=addr><label for=addr>Feed address</label><span class=row><input id=addr readonly value=\"{}\" spellcheck=false>\
let _ = write!(
b,
"</p><div class=show>{}<div class=meta><h2>{}</h2><p class=sub>{}{}</p>",
art(l, "art lg"),
esc(name(l)),
subscribers(l.subscribers),
l.subcategory.as_deref().map(|s| format!(" &middot; {}", esc(s))).unwrap_or_default()
);
let _ = write!(
b,
"<div class=addr><label for=addr>Feed address</label><span class=row><input id=addr readonly value=\"{}\" spellcheck=false>\
<button type=button data-copy=\"{}\">Copy</button></span><small>Paste it into any podcast app or feed reader to subscribe.</small></div></div></div>",
esc(&l.url), esc(&l.url));
esc(&l.url),
esc(&l.url)
);
if let Some(d) = p.description.as_deref().map(text).filter(|d| !d.is_empty() && d != name(l)) {
let _ = write!(b, "<p class=about>{}</p>", esc(&d));
}
@@ -199,17 +252,26 @@ async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Response {
None => esc(&title),
};
let blurb: String = e.description.as_deref().map(text).unwrap_or_default().chars().take(400).collect();
let when = [e.published.map(date), e.duration.filter(|d| *d > 0).map(clock)].into_iter().flatten().collect::<Vec<_>>().join(" &middot; ");
let _ = write!(b, "<article><b>{head}</b>{}{}</article>",
let when = [e.published.map(date), e.duration.filter(|d| *d > 0).map(clock)]
.into_iter()
.flatten()
.collect::<Vec<_>>()
.join(" &middot; ");
let _ = write!(
b,
"<article><b>{head}</b>{}{}</article>",
if blurb.is_empty() { String::new() } else { format!("<p>{}</p>", esc(&blurb)) },
if when.is_empty() { String::new() } else { format!("<small>{when}</small>") });
if when.is_empty() { String::new() } else { format!("<small>{when}</small>") }
);
}
page(name(l), Some(&text(p.description.as_deref().unwrap_or(""))), &b)
}
fn page(title: &str, description: Option<&str>, body: &str) -> Response {
let full = if title == "iPX Directory" { title.to_owned() } else { format!("{title} | iPX Directory") };
let desc = description.filter(|d| !d.is_empty()).map(|d| d.chars().take(200).collect::<String>())
let desc = description
.filter(|d| !d.is_empty())
.map(|d| d.chars().take(200).collect::<String>())
.unwrap_or_else(|| "Public podcasts and blogs that iPX servers share.".into());
let html = format!(
"<!doctype html><html lang=en><head><meta charset=utf-8><meta name=viewport content=\"width=device-width,initial-scale=1\">\
@@ -234,14 +296,22 @@ fn page(title: &str, description: Option<&str>, body: &str) -> Response {
}
fn not_found(why: &str) -> Response {
let mut r = page("Not found", None, &format!("<p class=back><a href=\"/\">Directory</a></p><h2>Not found</h2><p class=hint>{}</p>", esc(why)));
let mut r = page(
"Not found",
None,
&format!("<p class=back><a href=\"/\">Directory</a></p><h2>Not found</h2><p class=hint>{}</p>", esc(why)),
);
*r.status_mut() = StatusCode::NOT_FOUND;
r
}
fn failed(e: anyhow::Error) -> Response {
tracing::error!(error = %format!("{e:#}"), "a directory page failed");
let mut r = page("Unavailable", None, "<h2>Unavailable</h2><p class=hint>The directory could not read its list just now. Try again in a minute.</p>");
let mut r = page(
"Unavailable",
None,
"<h2>Unavailable</h2><p class=hint>The directory could not read its list just now. Try again in a minute.</p>",
);
*r.status_mut() = StatusCode::SERVICE_UNAVAILABLE;
r
}
@@ -253,8 +323,12 @@ fn redirect(to: &'static str) -> Response {
/// A cover from its publisher, over the feed's initials, which show where it has none or it fails.
fn art(l: &Listing, class: &str) -> String {
// The first letter of each of the first two words that have one: `<antirez>` is A, not nothing.
let initials: String = name(l).split_whitespace().filter_map(|w| w.chars().find(|c| c.is_alphanumeric())).take(2).collect();
let img = l.image.as_deref().filter(|u| u.starts_with("https://") || u.starts_with("http://"))
let initials: String =
name(l).split_whitespace().filter_map(|w| w.chars().find(|c| c.is_alphanumeric())).take(2).collect();
let img = l
.image
.as_deref()
.filter(|u| u.starts_with("https://") || u.starts_with("http://"))
.map(|u| format!("<img src=\"{}\" alt=\"\" loading=lazy referrerpolicy=no-referrer>", esc(u)))
.unwrap_or_default();
format!("<span class=\"{class}\" aria-hidden=true><i>{}</i>{img}</span>", esc(&initials.to_uppercase()))
@@ -301,7 +375,13 @@ fn clock(secs: i64) -> String {
fn text(html: &str) -> String {
// A space where each tag was, or two paragraphs run together: "drives.For over".
let stripped = ammonia::Builder::empty().clean(&html.replace('<', " <")).to_string();
let decoded = stripped.replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", "\"").replace("&#39;", "'").replace("&nbsp;", " ").replace("&amp;", "&");
let decoded = stripped
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&nbsp;", " ")
.replace("&amp;", "&");
decoded.split_whitespace().collect::<Vec<_>>().join(" ")
}

72
crates/directory/src/store.rs
View File

@@ -28,11 +28,7 @@ pub struct Due {
/// What a fetch found, to be stored.
pub enum Refreshed {
NotModified,
Feed {
parsed: ipx_feed::ParsedFeed,
etag: Option<String>,
last_modified: Option<String>,
},
Feed { parsed: ipx_feed::ParsedFeed, etag: Option<String>, last_modified: Option<String> },
Failed(String),
}
@@ -122,7 +118,12 @@ impl Store {
let schema = format!("ipxd_{pid}_{n}");
admin.db.execute_unprepared(&format!("CREATE SCHEMA {schema}")).await.unwrap();
let sep = if url.contains('?') { '&' } else { '?' };
Self::open(&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning"), false).await.unwrap()
Self::open(
&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning"),
false,
)
.await
.unwrap()
}
/// `keep_open` for the server, which keeps its whole pool connected.
@@ -195,8 +196,12 @@ impl Store {
/// blocked feed is left out and counted; the rest are added to the feeds, new ones to be
/// fetched by the refresher. Feeds no instance sends any more go.
pub async fn snapshot(&self, instance: &str, entries: &[SnapshotEntry]) -> Result<SnapshotResult> {
let blocked: std::collections::HashSet<String> =
self.rows("SELECT id FROM blocked", vec![]).await?.iter().map(|r| r.try_get("", "id")).collect::<Result<_, _>>()?;
let blocked: std::collections::HashSet<String> = self
.rows("SELECT id FROM blocked", vec![])
.await?
.iter()
.map(|r| r.try_get("", "id"))
.collect::<Result<_, _>>()?;
let mut result = SnapshotResult::default();
let mut keep = std::collections::BTreeMap::new();
for e in entries {
@@ -372,7 +377,13 @@ impl Store {
)
.await?
.iter()
.map(|r| Ok(Instance { name: r.try_get("", "instance")?, feeds: r.try_get("", "feeds")?, subscribers: r.try_get("", "subscribers")? }))
.map(|r| {
Ok(Instance {
name: r.try_get("", "instance")?,
feeds: r.try_get("", "feeds")?,
subscribers: r.try_get("", "subscribers")?,
})
})
.collect()
}
@@ -380,7 +391,14 @@ impl Store {
self.rows("SELECT id, url, title, at FROM blocked ORDER BY at DESC, id", vec![])
.await?
.iter()
.map(|r| Ok(Blocked { id: r.try_get("", "id")?, url: r.try_get("", "url")?, title: r.try_get("", "title")?, at: r.try_get("", "at")? }))
.map(|r| {
Ok(Blocked {
id: r.try_get("", "id")?,
url: r.try_get("", "url")?,
title: r.try_get("", "title")?,
at: r.try_get("", "at")?,
})
})
.collect()
}
@@ -425,7 +443,11 @@ impl Store {
let t = now();
match got {
Refreshed::NotModified => {
self.exec("UPDATE feeds SET checked_at = $1, last_error = NULL, error_since = NULL WHERE id = $2", vec![t.into(), id.into()]).await?;
self.exec(
"UPDATE feeds SET checked_at = $1, last_error = NULL, error_since = NULL WHERE id = $2",
vec![t.into(), id.into()],
)
.await?;
}
Refreshed::Failed(err) => {
self.exec(
@@ -436,16 +458,25 @@ impl Store {
}
Refreshed::Feed { parsed, etag, last_modified } => {
let path = parsed.category.as_deref().map(ipx_feed::category_path);
let podcast = parsed.entries.iter().flat_map(|e| &e.enclosures).any(|e| {
e.mime.as_deref().is_some_and(|m| m.starts_with("audio/") || m.starts_with("video/"))
});
let podcast = parsed
.entries
.iter()
.flat_map(|e| &e.enclosures)
.any(|e| e.mime.as_deref().is_some_and(|m| m.starts_with("audio/") || m.starts_with("video/")));
let mut entries = parsed.entries;
entries.sort_by_key(|e| std::cmp::Reverse(e.published));
// What an item is, never its files: their addresses are where a key would be.
let items: Vec<Item> = entries
.into_iter()
.take(ITEMS)
.map(|e| Item { guid: e.guid, title: e.title, link: e.link, published: e.published, duration: e.duration, description: e.description })
.map(|e| Item {
guid: e.guid,
title: e.title,
link: e.link,
published: e.published,
duration: e.duration,
description: e.description,
})
.collect();
self.exec(
"UPDATE feeds SET title = $1, image = $2, category = $3, subcategory = $4, description = $5,
@@ -541,7 +572,10 @@ pub(crate) mod tests {
assert_eq!(all[0].subscribers, 3);
let i = s.instances().await.unwrap();
assert_eq!(i.iter().map(|i| (i.name.as_str(), i.feeds, i.subscribers)).collect::<Vec<_>>(), [("a", 1, 1), ("b", 1, 2)]);
assert_eq!(
i.iter().map(|i| (i.name.as_str(), i.feeds, i.subscribers)).collect::<Vec<_>>(),
[("a", 1, 1), ("b", 1, 2)]
);
// A token taken out of the config file takes its counts.
s.keep_instances(&["b".into()]).await.unwrap();
@@ -583,7 +617,11 @@ pub(crate) mod tests {
#[tokio::test]
async fn a_feed_naming_no_category_takes_the_one_a_server_gave_it() {
let s = Store::scratch().await;
let entry = |c: Option<&str>| SnapshotEntry { url: "https://x.test/blog.xml".into(), subscribers: 1, category: c.map(Into::into) };
let entry = |c: Option<&str>| SnapshotEntry {
url: "https://x.test/blog.xml".into(),
subscribers: 1,
category: c.map(Into::into),
};
s.snapshot("a", &[entry(Some("Visual Arts"))]).await.unwrap();
let d = s.due(0, 10).await.unwrap().remove(0);
let mut parsed = ipx_feed::parse(include_bytes!("../../../tests/data/rss2.xml")).unwrap();