ipx-directory: the directory of public podcasts ipx servers share (#204)
A new crate and service, crates/directory, live at ipx-dir.sdf1.net. ipx servers send the public feeds their people subscribe to (POST /v1/snapshot, replacing what that token sent before); the directory fetches each feed's title, artwork, category, description and latest twenty items every six hours with ipx-feed, and lists them A to Z and by subscribers summed over every server. It holds metadata only, never a file or a file's address. Tokens are in its config file as SHA-256 hashes, each ro, rw or rwa, and the file is read again within a minute of a change, so adding and revoking one is an edit and nothing else; a revoked token takes its server's counts. At first they were to come from environment variables, which would have meant recreating the container to add one. A file that does not load keeps the tokens as they were. Every address sent is checked with ipx_feed::url_looks_private, the check ipx makes before sending, so one server's miss does not publish a paid feed to every other; an rwa token can remove and block what both miss. Deployed: ipx-directory in the ipx Arcane project, database ipx_directory (its own login) on the databases project's Postgres, which db-backup already covers, and a tunnel route with no Access in front. The store's tests run on Postgres as well as SQLite when IPX_TEST_DATABASE_URL is set. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
27
crates/directory/Cargo.toml
Normal file
27
crates/directory/Cargo.toml
Normal file
@@ -0,0 +1,27 @@
|
||||
[package]
|
||||
name = "ipx-directory"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "ipx-directory"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
axum.workspace = true
|
||||
clap.workspace = true
|
||||
ipx-feed.workspace = true
|
||||
reqwest.workspace = true
|
||||
sea-orm.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
tokio.workspace = true
|
||||
toml.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
url.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tower.workspace = true
|
||||
172
crates/directory/src/config.rs
Normal file
172
crates/directory/src/config.rs
Normal file
@@ -0,0 +1,172 @@
|
||||
//! The directory's config file: where it listens, where its database is, and the tokens it
|
||||
//! accepts. Tokens are only ever changed here, never over the API, so whoever can edit this
|
||||
//! file decides who gets in (#204).
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde::Deserialize;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Config {
|
||||
#[serde(default = "default_bind")]
|
||||
pub bind: String,
|
||||
/// The environment variable holding the database URL, so the file holds no secret.
|
||||
#[serde(default = "default_database_url_env")]
|
||||
pub database_url_env: String,
|
||||
#[serde(default)]
|
||||
pub tokens: Vec<TokenCfg>,
|
||||
}
|
||||
|
||||
fn default_bind() -> String {
|
||||
"0.0.0.0:8100".into()
|
||||
}
|
||||
|
||||
fn default_database_url_env() -> String {
|
||||
"DIRECTORY_DATABASE_URL".into()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TokenCfg {
|
||||
/// Who holds it: what its snapshot is filed under and what the log says, never the token.
|
||||
pub name: String,
|
||||
/// No default: a token whose access is missing or misspelled is refused when the file is
|
||||
/// read, rather than given some access nobody chose.
|
||||
pub access: Access,
|
||||
/// The token's SHA-256, in hex: what `ipx-directory token new` prints. The file holds no
|
||||
/// token, so adding or revoking one is an edit here and nothing else.
|
||||
pub token_sha256: String,
|
||||
}
|
||||
|
||||
/// What a token may do, each level including the one before.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, serde::Serialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Access {
|
||||
/// Browse.
|
||||
Ro,
|
||||
/// Browse and send snapshots.
|
||||
Rw,
|
||||
/// All that, and remove, block and refresh feeds and drop an instance's snapshot.
|
||||
Rwa,
|
||||
}
|
||||
|
||||
/// A token as the server checks it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Token {
|
||||
pub name: String,
|
||||
pub access: Access,
|
||||
/// SHA-256 of the token, lower-case hex.
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn load(path: &Path) -> Result<Self> {
|
||||
let text = std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
|
||||
Self::parse(&text).with_context(|| format!("in {}", path.display()))
|
||||
}
|
||||
|
||||
pub fn parse(text: &str) -> Result<Self> {
|
||||
let cfg: Config = toml::from_str(text)?;
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for t in &cfg.tokens {
|
||||
if t.name.trim().is_empty() {
|
||||
bail!("a token has an empty name");
|
||||
}
|
||||
// Snapshots are filed by name: two instances under one name would overwrite each other.
|
||||
if !seen.insert(t.name.as_str()) {
|
||||
bail!("two tokens are named {:?}", t.name);
|
||||
}
|
||||
}
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// The tokens as the server checks them. A hash that is not 64 hex digits, or the same
|
||||
/// hash under two names, is an error: which name a request is would be ambiguous.
|
||||
pub fn tokens(&self) -> Result<Vec<Token>> {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
self.tokens
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let sha256 = t.token_sha256.trim().to_ascii_lowercase();
|
||||
if sha256.len() != 64 || !sha256.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
bail!("token {:?}: token_sha256 is not a SHA-256 in hex", t.name);
|
||||
}
|
||||
if !seen.insert(sha256.clone()) {
|
||||
bail!("token {:?} is the same token as another", t.name);
|
||||
}
|
||||
Ok(Token { name: t.name.clone(), access: t.access, sha256 })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// A token's SHA-256 in hex, as the config file holds it.
|
||||
pub fn hash(token: &str) -> String {
|
||||
use sha2::Digest;
|
||||
sha2::Sha256::digest(token.trim().as_bytes()).iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Equal without saying, by how long it took, how much of the guess was right.
|
||||
pub fn same_secret(a: &str, b: &str) -> bool {
|
||||
let (a, b) = (a.as_bytes(), b.as_bytes());
|
||||
a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0
|
||||
}
|
||||
|
||||
/// A new token: `ipxd_` and 256 random bits, the prefix so one found in a log says what it is.
|
||||
pub fn new_token() -> String {
|
||||
use std::io::Read;
|
||||
let mut bytes = [0u8; 32];
|
||||
// Falling back to the clock would make a guessable token. Better to fail.
|
||||
std::fs::File::open("/dev/urandom")
|
||||
.and_then(|mut f| f.read_exact(&mut bytes))
|
||||
.expect("no source of randomness for a token");
|
||||
format!("ipxd_{}", bytes.iter().map(|b| format!("{b:02x}")).collect::<String>())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const GOOD: &str = r#"
|
||||
[[tokens]]
|
||||
name = "tower"
|
||||
access = "rwa"
|
||||
token_sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
[[tokens]]
|
||||
name = "friend"
|
||||
access = "ro"
|
||||
token_sha256 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn tokens_carry_their_access_and_take_their_value_from_the_environment() {
|
||||
let cfg = Config::parse(GOOD).unwrap();
|
||||
assert_eq!(cfg.bind, "0.0.0.0:8100");
|
||||
let t = cfg.tokens().unwrap();
|
||||
assert_eq!((t[0].name.as_str(), t[0].access, t[0].sha256.as_str()), ("tower", Access::Rwa, "a".repeat(64).as_str()));
|
||||
assert_eq!(t[1].access, Access::Ro);
|
||||
assert!(Access::Ro < Access::Rw && Access::Rw < Access::Rwa);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_misspelled_access_a_reused_name_or_a_bad_hash_is_refused() {
|
||||
let h = "a".repeat(64);
|
||||
assert!(Config::parse(&format!("[[tokens]]\nname = \"a\"\ntoken_sha256 = \"{h}\"")).is_err());
|
||||
assert!(Config::parse(&format!("[[tokens]]\nname = \"a\"\naccess = \"admin\"\ntoken_sha256 = \"{h}\"")).is_err());
|
||||
assert!(Config::parse(&GOOD.replace("\"friend\"", "\"tower\"")).is_err());
|
||||
assert!(Config::parse(&GOOD.replace(&"b".repeat(64), "xyz")).unwrap().tokens().is_err());
|
||||
assert!(Config::parse(&GOOD.replace(&"b".repeat(64), &h)).unwrap().tokens().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secrets_compare_whole() {
|
||||
assert!(same_secret("abc", "abc"));
|
||||
assert!(!same_secret("abc", "abd"));
|
||||
assert!(!same_secret("abc", "abcd"));
|
||||
let t = new_token();
|
||||
assert!(t.starts_with("ipxd_") && t.len() == 69 && t != new_token());
|
||||
assert_eq!(hash("abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
|
||||
}
|
||||
}
|
||||
462
crates/directory/src/lib.rs
Normal file
462
crates/directory/src/lib.rs
Normal file
@@ -0,0 +1,462 @@
|
||||
//! ipx's directory service (#204): a catalogue of public podcasts and blogs that ipx servers
|
||||
//! share. It holds metadata only, never a file: each server still fetches its own feeds. ipx
|
||||
//! servers send which public feeds their people subscribe to, the directory fetches each
|
||||
//! feed's title, artwork, description and latest items on a slow schedule, and any server
|
||||
//! holding a token can browse it.
|
||||
//!
|
||||
//! The types here are the wire format, shared with ipx, which asks the directory on its
|
||||
//! people's behalf.
|
||||
|
||||
pub mod config;
|
||||
pub mod store;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use axum::extract::{Path, Query, Request, State};
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::middleware::Next;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{delete, get, post};
|
||||
use axum::{Extension, Json, Router};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use config::{Access, Token};
|
||||
use store::{Refreshed, Store};
|
||||
|
||||
/// A feed as the directory lists it.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Listing {
|
||||
/// The first 64 bits of the SHA-256 of its normalised address (`store::feed_id`).
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
pub title: Option<String>,
|
||||
pub image: Option<String>,
|
||||
/// Apple's category and the subcategory within it, from the feed's own iTunes category.
|
||||
pub category: Option<String>,
|
||||
pub subcategory: Option<String>,
|
||||
/// Any audio or video in its latest items.
|
||||
pub podcast: bool,
|
||||
/// Summed over every instance's latest snapshot.
|
||||
pub subscribers: i64,
|
||||
}
|
||||
|
||||
/// One feed's page: what it says it is, and its latest items.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct FeedPage {
|
||||
#[serde(flatten)]
|
||||
pub listing: Listing,
|
||||
/// The feed's own text, HTML as often as not, as it was sent: whoever shows it sanitizes it.
|
||||
pub description: Option<String>,
|
||||
/// Why its last check failed, in words, when it did.
|
||||
pub failing: Option<String>,
|
||||
pub items: Vec<Item>,
|
||||
}
|
||||
|
||||
/// One of a feed's latest items: what it is, never its files.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Item {
|
||||
pub guid: String,
|
||||
pub title: Option<String>,
|
||||
pub link: Option<String>,
|
||||
pub published: Option<i64>,
|
||||
pub duration: Option<i64>,
|
||||
/// As the feed sent it, unsanitized.
|
||||
pub description: Option<String>,
|
||||
}
|
||||
|
||||
/// One feed in a snapshot: its address, and how many people on the sending server subscribe.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SnapshotEntry {
|
||||
pub url: String,
|
||||
pub subscribers: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct SnapshotResult {
|
||||
pub accepted: usize,
|
||||
/// Addresses that look like they carry a subscriber's own access.
|
||||
pub refused_private: usize,
|
||||
/// Feeds an admin removed from the directory.
|
||||
pub refused_blocked: usize,
|
||||
/// Not an http(s) address.
|
||||
pub refused_invalid: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Me {
|
||||
pub name: String,
|
||||
pub access: Access,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Blocked {
|
||||
pub id: String,
|
||||
pub url: Option<String>,
|
||||
pub title: Option<String>,
|
||||
pub at: i64,
|
||||
}
|
||||
|
||||
/// The most a snapshot may list: far more than one server's subscriptions, few enough that one
|
||||
/// request cannot hold the database for long.
|
||||
pub const MAX_SNAPSHOT: usize = 20_000;
|
||||
|
||||
/// How often each feed is fetched again. Metadata changes slowly, and every server subscribed
|
||||
/// to a feed fetches it itself anyway.
|
||||
pub const REFRESH_EVERY: i64 = 6 * 3600;
|
||||
|
||||
pub struct Directory {
|
||||
pub store: Store,
|
||||
tokens: RwLock<Vec<Token>>,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
pub type Shared = Arc<Directory>;
|
||||
|
||||
impl Directory {
|
||||
pub fn new(store: Store, tokens: Vec<Token>) -> Result<Shared, reqwest::Error> {
|
||||
// No redirects: ipx_feed::fetch follows them itself, so it sees each hop.
|
||||
let client = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.user_agent(concat!("ipx-directory/", env!("CARGO_PKG_VERSION")))
|
||||
.build()?;
|
||||
Ok(Arc::new(Self { store, tokens: RwLock::new(tokens), client }))
|
||||
}
|
||||
|
||||
/// Takes the tokens of a config file read again, and drops the snapshots of any that went.
|
||||
pub async fn set_tokens(&self, tokens: Vec<Token>) -> anyhow::Result<u64> {
|
||||
let names: Vec<String> = tokens.iter().map(|t| t.name.clone()).collect();
|
||||
*self.tokens.write().unwrap() = tokens;
|
||||
self.store.keep_instances(&names).await
|
||||
}
|
||||
|
||||
fn caller(&self, presented: &str) -> Option<Me> {
|
||||
let h = config::hash(presented);
|
||||
// Every token compared, whether or not one matched, so timing says nothing either way.
|
||||
self.tokens
|
||||
.read()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.fold(None, |found, t| if config::same_secret(&t.sha256, &h) { Some(Me { name: t.name.clone(), access: t.access }) } else { found })
|
||||
}
|
||||
|
||||
/// Fetches one feed and stores what it found.
|
||||
pub async fn refresh(&self, due: &store::Due) -> anyhow::Result<()> {
|
||||
let got = match ipx_feed::fetch(&self.client, &due.url, None, due.etag.as_deref(), due.last_modified.as_deref()).await {
|
||||
Ok((ipx_feed::Fetched::NotModified, _)) => Refreshed::NotModified,
|
||||
Ok((ipx_feed::Fetched::Body { bytes, etag, last_modified }, _)) => {
|
||||
if ipx_feed::is_opml(&bytes) {
|
||||
// ipx sends the feeds inside an OPML, never the OPML.
|
||||
Refreshed::Failed("an OPML list, not a feed".into())
|
||||
} else {
|
||||
match ipx_feed::parse(&bytes) {
|
||||
Ok(parsed) => Refreshed::Feed { parsed, etag, last_modified },
|
||||
Err(e) => Refreshed::Failed(format!("{e:#}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => Refreshed::Failed(format!("{e:#}")),
|
||||
};
|
||||
if let Refreshed::Failed(e) = &got {
|
||||
tracing::info!(feed = %due.id, error = %e, "feed check failed");
|
||||
}
|
||||
// ponytail: a feed that moved for good is still asked for at its old address, which
|
||||
// redirects; follow the move when a feed's id can change without losing its counts.
|
||||
self.store.save(&due.id, got).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches whatever is due, a few at a time, forever.
|
||||
pub async fn refresher(dir: Shared) {
|
||||
loop {
|
||||
match dir.store.due(REFRESH_EVERY, 25).await {
|
||||
Ok(due) => {
|
||||
for d in &due {
|
||||
if let Err(e) = dir.refresh(d).await {
|
||||
tracing::warn!(feed = %d.id, error = %format!("{e:#}"), "could not store a feed check");
|
||||
}
|
||||
}
|
||||
if !due.is_empty() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %format!("{e:#}"), "could not read which feeds are due"),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the config file again whenever it changes, checked each minute. A file that fails to
|
||||
/// load leaves the tokens as they were, so a typo locks nobody out.
|
||||
pub async fn reloader(dir: Shared, path: std::path::PathBuf) {
|
||||
let mtime = |p: &std::path::Path| std::fs::metadata(p).and_then(|m| m.modified()).ok();
|
||||
let mut last = mtime(&path);
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
let now = mtime(&path);
|
||||
if now == last {
|
||||
continue;
|
||||
}
|
||||
last = now;
|
||||
match config::Config::load(&path).and_then(|c| c.tokens()) {
|
||||
Ok(tokens) => {
|
||||
let n = tokens.len();
|
||||
match dir.set_tokens(tokens).await {
|
||||
Ok(dropped) => tracing::info!(tokens = n, dropped, "config read again"),
|
||||
Err(e) => tracing::warn!(error = %format!("{e:#}"), "config read again, but old snapshots could not be dropped"),
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %format!("{e:#}"), "config changed but does not load; keeping the tokens as they were"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn router(dir: Shared) -> Router {
|
||||
let v1 = Router::new()
|
||||
.route("/v1/me", get(me))
|
||||
.route("/v1/feeds", get(list))
|
||||
.route("/v1/popular", get(popular))
|
||||
.route("/v1/feeds/{id}", get(feed).delete(remove))
|
||||
.route("/v1/feeds/{id}/refresh", post(refresh))
|
||||
.route("/v1/snapshot", post(snapshot))
|
||||
.route("/v1/blocked", get(blocked))
|
||||
.route("/v1/blocked/{id}", delete(unblock))
|
||||
.route("/v1/instances/{name}", delete(drop_instance))
|
||||
.layer(axum::middleware::from_fn_with_state(dir.clone(), authenticate));
|
||||
Router::new()
|
||||
.route("/healthz", get(healthz))
|
||||
.merge(v1)
|
||||
.layer(axum::middleware::from_fn(access_log))
|
||||
.with_state(dir)
|
||||
}
|
||||
|
||||
struct ApiError(StatusCode, String);
|
||||
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> Response {
|
||||
(self.0, Json(serde_json::json!({ "error": self.1 }))).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<anyhow::Error> for ApiError {
|
||||
fn from(e: anyhow::Error) -> Self {
|
||||
tracing::error!(error = %format!("{e:#}"), "request failed");
|
||||
ApiError(StatusCode::INTERNAL_SERVER_ERROR, "something went wrong; the directory's log says what".into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Who made the request, for the access log, set by `authenticate`.
|
||||
#[derive(Clone)]
|
||||
struct Who(String);
|
||||
|
||||
/// One line a request, with the token's name and never the token: what the Grafana panels read.
|
||||
async fn access_log(req: Request, next: Next) -> Response {
|
||||
let (method, path) = (req.method().clone(), req.uri().path().to_owned());
|
||||
let start = Instant::now();
|
||||
let resp = next.run(req).await;
|
||||
let who = resp.extensions().get::<Who>().map_or("", |w| w.0.as_str()).to_owned();
|
||||
let status = resp.status().as_u16();
|
||||
if path != "/healthz" {
|
||||
tracing::info!(
|
||||
target: "ipx_directory::http",
|
||||
method = %method, path = %path, status, duration_ms = start.elapsed().as_millis() as u64, token = %who,
|
||||
"{method} {path} -> {status}"
|
||||
);
|
||||
}
|
||||
resp
|
||||
}
|
||||
|
||||
async fn authenticate(State(dir): State<Shared>, mut req: Request, next: Next) -> Response {
|
||||
let presented = req
|
||||
.headers()
|
||||
.get(header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.map(str::trim)
|
||||
.filter(|t| !t.is_empty());
|
||||
let Some(me) = presented.and_then(|t| dir.caller(t)) else {
|
||||
return ApiError(StatusCode::UNAUTHORIZED, "a valid token is needed: Authorization: Bearer <token>".into()).into_response();
|
||||
};
|
||||
let who = Who(me.name.clone());
|
||||
req.extensions_mut().insert(me);
|
||||
let mut resp = next.run(req).await;
|
||||
resp.extensions_mut().insert(who);
|
||||
resp
|
||||
}
|
||||
|
||||
fn need(me: &Me, access: Access) -> Result<(), ApiError> {
|
||||
if me.access >= access {
|
||||
Ok(())
|
||||
} else {
|
||||
let what = match access {
|
||||
Access::Ro => "browse",
|
||||
Access::Rw => "send snapshots",
|
||||
Access::Rwa => "administer the directory",
|
||||
};
|
||||
Err(ApiError(StatusCode::FORBIDDEN, format!("this token may not {what}")))
|
||||
}
|
||||
}
|
||||
|
||||
async fn healthz(State(dir): State<Shared>) -> Result<&'static str, ApiError> {
|
||||
dir.store.ping().await?;
|
||||
Ok("ok")
|
||||
}
|
||||
|
||||
async fn me(Extension(me): Extension<Me>) -> Json<Me> {
|
||||
Json(me)
|
||||
}
|
||||
|
||||
async fn list(State(dir): State<Shared>) -> Result<Json<Vec<Listing>>, ApiError> {
|
||||
Ok(Json(dir.store.list().await?))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Limit {
|
||||
limit: Option<u64>,
|
||||
}
|
||||
|
||||
async fn popular(State(dir): State<Shared>, Query(q): Query<Limit>) -> Result<Json<Vec<Listing>>, ApiError> {
|
||||
Ok(Json(dir.store.popular(q.limit.unwrap_or(10).clamp(1, 100)).await?))
|
||||
}
|
||||
|
||||
async fn feed(State(dir): State<Shared>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
|
||||
dir.store.feed(&id).await?.map(Json).ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))
|
||||
}
|
||||
|
||||
async fn snapshot(
|
||||
State(dir): State<Shared>,
|
||||
Extension(me): Extension<Me>,
|
||||
Json(entries): Json<Vec<SnapshotEntry>>,
|
||||
) -> Result<Json<SnapshotResult>, ApiError> {
|
||||
need(&me, Access::Rw)?;
|
||||
if entries.len() > MAX_SNAPSHOT {
|
||||
return Err(ApiError(StatusCode::PAYLOAD_TOO_LARGE, format!("a snapshot lists at most {MAX_SNAPSHOT} feeds")));
|
||||
}
|
||||
let r = dir.store.snapshot(&me.name, &entries).await?;
|
||||
tracing::info!(
|
||||
instance = %me.name, accepted = r.accepted, refused_private = r.refused_private,
|
||||
refused_blocked = r.refused_blocked, refused_invalid = r.refused_invalid, "snapshot"
|
||||
);
|
||||
Ok(Json(r))
|
||||
}
|
||||
|
||||
async fn remove(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
|
||||
need(&me, Access::Rwa)?;
|
||||
dir.store.block(&id).await?;
|
||||
tracing::info!(feed = %id, by = %me.name, "feed removed and blocked");
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
async fn blocked(State(dir): State<Shared>, Extension(me): Extension<Me>) -> Result<Json<Vec<Blocked>>, ApiError> {
|
||||
need(&me, Access::Rwa)?;
|
||||
Ok(Json(dir.store.blocked().await?))
|
||||
}
|
||||
|
||||
async fn unblock(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<StatusCode, ApiError> {
|
||||
need(&me, Access::Rwa)?;
|
||||
if !dir.store.unblock(&id).await? {
|
||||
return Err(ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not blocked")));
|
||||
}
|
||||
tracing::info!(feed = %id, by = %me.name, "feed unblocked");
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
async fn drop_instance(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(name): Path<String>) -> Result<StatusCode, ApiError> {
|
||||
need(&me, Access::Rwa)?;
|
||||
let n = dir.store.drop_instance(&name).await?;
|
||||
tracing::info!(instance = %name, feeds = n, by = %me.name, "snapshot dropped");
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
async fn refresh(State(dir): State<Shared>, Extension(me): Extension<Me>, Path(id): Path<String>) -> Result<Json<FeedPage>, ApiError> {
|
||||
need(&me, Access::Rwa)?;
|
||||
let due = dir.store.due_one(&id).await?.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, format!("{id:?} is not in the directory")))?;
|
||||
dir.refresh(&due).await?;
|
||||
feed(State(dir), Path(id)).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::body::Body;
|
||||
use tower::ServiceExt;
|
||||
|
||||
async fn app() -> Router {
|
||||
let tokens = ["ro", "rw", "rwa"]
|
||||
.iter()
|
||||
.map(|n| Token {
|
||||
name: format!("{n}-token"),
|
||||
access: serde_json::from_value(serde_json::json!(n)).unwrap(),
|
||||
sha256: config::hash(&format!("secret-{n}")),
|
||||
})
|
||||
.collect();
|
||||
router(Directory::new(Store::scratch().await, tokens).unwrap())
|
||||
}
|
||||
|
||||
async fn call(app: &Router, method: &str, path: &str, token: Option<&str>, body: Option<serde_json::Value>) -> (u16, serde_json::Value) {
|
||||
let mut req = axum::http::Request::builder().method(method).uri(path);
|
||||
if let Some(t) = token {
|
||||
req = req.header("authorization", format!("Bearer secret-{t}"));
|
||||
}
|
||||
let req = match body {
|
||||
Some(b) => req.header("content-type", "application/json").body(Body::from(b.to_string())),
|
||||
None => req.body(Body::empty()),
|
||||
}
|
||||
.unwrap();
|
||||
let resp = app.clone().oneshot(req).await.unwrap();
|
||||
let status = resp.status().as_u16();
|
||||
let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap();
|
||||
(status, serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn each_access_level_does_what_it_may_and_no_more() {
|
||||
let app = app().await;
|
||||
let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 2 }]);
|
||||
|
||||
assert_eq!(call(&app, "GET", "/v1/feeds", None, None).await.0, 401);
|
||||
assert_eq!(call(&app, "GET", "/v1/feeds", Some("wrong"), None).await.0, 401);
|
||||
assert_eq!(call(&app, "GET", "/healthz", None, None).await.0, 200);
|
||||
|
||||
let (s, me) = call(&app, "GET", "/v1/me", Some("ro"), None).await;
|
||||
assert_eq!((s, me["name"].as_str(), me["access"].as_str()), (200, Some("ro-token"), Some("ro")));
|
||||
assert_eq!(call(&app, "POST", "/v1/snapshot", Some("ro"), Some(snap.clone())).await.0, 403);
|
||||
|
||||
let (s, r) = call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap.clone())).await;
|
||||
assert_eq!((s, r["accepted"].as_u64()), (200, Some(1)));
|
||||
let (_, list) = call(&app, "GET", "/v1/feeds", Some("ro"), None).await;
|
||||
let id = list[0]["id"].as_str().unwrap().to_owned();
|
||||
assert_eq!(list[0]["subscribers"], 2);
|
||||
assert_eq!(call(&app, "GET", &format!("/v1/feeds/{id}"), Some("ro"), None).await.0, 200);
|
||||
assert_eq!(call(&app, "GET", "/v1/feeds/nope", Some("ro"), None).await.0, 404);
|
||||
assert_eq!(call(&app, "GET", "/v1/popular?limit=5", Some("ro"), None).await.1[0]["id"], id.as_str());
|
||||
|
||||
for (m, p) in [("DELETE", format!("/v1/feeds/{id}")), ("GET", "/v1/blocked".into()), ("DELETE", "/v1/instances/rw-token".into())] {
|
||||
assert_eq!(call(&app, m, &p, Some("rw"), None).await.0, 403, "{m} {p}");
|
||||
}
|
||||
assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), Some("rwa"), None).await.0, 204);
|
||||
assert_eq!(call(&app, "GET", "/v1/blocked", Some("rwa"), None).await.1[0]["id"], id.as_str());
|
||||
assert_eq!(call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap.clone())).await.1["refused_blocked"], 1);
|
||||
assert_eq!(call(&app, "DELETE", &format!("/v1/blocked/{id}"), Some("rwa"), None).await.0, 204);
|
||||
assert_eq!(call(&app, "DELETE", &format!("/v1/blocked/{id}"), Some("rwa"), None).await.0, 404);
|
||||
call(&app, "POST", "/v1/snapshot", Some("rw"), Some(snap)).await;
|
||||
assert_eq!(call(&app, "DELETE", "/v1/instances/rw-token", Some("rwa"), None).await.0, 204);
|
||||
assert_eq!(call(&app, "GET", "/v1/feeds", Some("ro"), None).await.1, serde_json::json!([]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_revoked_token_loses_its_access_and_its_counts() {
|
||||
let dir = Directory::new(Store::scratch().await, vec![]).unwrap();
|
||||
let t = |n: &str| Token { name: n.into(), access: Access::Rw, sha256: config::hash(&format!("secret-{n}")) };
|
||||
dir.set_tokens(vec![t("a"), t("b")]).await.unwrap();
|
||||
let app = router(dir.clone());
|
||||
let snap = serde_json::json!([{ "url": "https://x.test/a.xml", "subscribers": 1 }]);
|
||||
call(&app, "POST", "/v1/snapshot", Some("a"), Some(snap.clone())).await;
|
||||
call(&app, "POST", "/v1/snapshot", Some("b"), Some(snap)).await;
|
||||
assert_eq!(call(&app, "GET", "/v1/feeds", Some("a"), None).await.1[0]["subscribers"], 2);
|
||||
|
||||
dir.set_tokens(vec![t("a")]).await.unwrap();
|
||||
assert_eq!(call(&app, "GET", "/v1/feeds", Some("b"), None).await.0, 401);
|
||||
assert_eq!(call(&app, "GET", "/v1/feeds", Some("a"), None).await.1[0]["subscribers"], 1);
|
||||
}
|
||||
}
|
||||
109
crates/directory/src/main.rs
Normal file
109
crates/directory/src/main.rs
Normal file
@@ -0,0 +1,109 @@
|
||||
//! `ipx-directory`: serve the directory, make a token, or check that a running one answers.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::{Parser, Subcommand};
|
||||
use ipx_directory::{Directory, config, store::Store};
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(version, about = "ipx's directory of public podcasts")]
|
||||
struct Cli {
|
||||
/// The config file: where to listen, the database, and the tokens it accepts. Otherwise
|
||||
/// DIRECTORY_CONFIG, or else /config/directory.toml.
|
||||
#[arg(long)]
|
||||
config: Option<std::path::PathBuf>,
|
||||
#[command(subcommand)]
|
||||
cmd: Option<Cmd>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Cmd {
|
||||
/// Serve the directory (the default).
|
||||
Serve,
|
||||
/// Make a token, and print it with the lines that let it in.
|
||||
Token {
|
||||
#[command(subcommand)]
|
||||
cmd: TokenCmd,
|
||||
},
|
||||
/// Check that the directory on this machine answers and can read its database: the
|
||||
/// container's healthcheck.
|
||||
Health,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum TokenCmd {
|
||||
New {
|
||||
/// Who it is for, as the log and their snapshot will name them.
|
||||
name: String,
|
||||
#[arg(long, value_parser = ["ro", "rw", "rwa"])]
|
||||
access: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
let config_path = cli
|
||||
.config
|
||||
.or_else(|| std::env::var_os("DIRECTORY_CONFIG").map(Into::into))
|
||||
.unwrap_or_else(|| "/config/directory.toml".into());
|
||||
match cli.cmd.unwrap_or(Cmd::Serve) {
|
||||
Cmd::Serve => serve(&config_path).await,
|
||||
Cmd::Token { cmd: TokenCmd::New { name, access } } => {
|
||||
let token = config::new_token();
|
||||
println!("A token for {name}, to give them; it is not kept anywhere:\n\n {token}\n");
|
||||
println!("To let it in, add this to {}:\n", config_path.display());
|
||||
println!("[[tokens]]\nname = {name:?}\naccess = {access:?}\ntoken_sha256 = {:?}", config::hash(&token));
|
||||
println!("\nThe directory reads the file again within a minute. Delete the entry to revoke it.");
|
||||
Ok(())
|
||||
}
|
||||
Cmd::Health => {
|
||||
let cfg = config::Config::load(&config_path)?;
|
||||
let port = cfg.bind.rsplit(':').next().unwrap_or("8100");
|
||||
let resp = reqwest::Client::new()
|
||||
.get(format!("http://127.0.0.1:{port}/healthz"))
|
||||
.timeout(std::time::Duration::from_secs(4))
|
||||
.send()
|
||||
.await
|
||||
.context("the directory does not answer")?;
|
||||
anyhow::ensure!(resp.status().is_success(), "the directory answers {}", resp.status());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn serve(path: &std::path::Path) -> Result<()> {
|
||||
// One JSON object a line for Loki when DIRECTORY_LOG_FORMAT=json, as ipx does; text otherwise.
|
||||
let json = std::env::var("DIRECTORY_LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
|
||||
let filter = tracing_subscriber::EnvFilter::try_from_env("DIRECTORY_LOG").unwrap_or_else(|_| "ipx_directory=info".into());
|
||||
tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with((!json).then(|| tracing_subscriber::fmt::layer().with_writer(std::io::stderr)))
|
||||
.with(json.then(|| tracing_subscriber::fmt::layer().json().flatten_event(true).with_writer(std::io::stderr)))
|
||||
.init();
|
||||
|
||||
let cfg = config::Config::load(path)?;
|
||||
let tokens = cfg.tokens()?;
|
||||
let url = std::env::var(&cfg.database_url_env)
|
||||
.ok()
|
||||
.filter(|u| !u.trim().is_empty())
|
||||
.with_context(|| format!("{} is not set: the directory needs a database URL", cfg.database_url_env))?;
|
||||
let store = Store::open(&url).await.context("opening the database")?;
|
||||
let dir = Directory::new(store, vec![])?;
|
||||
let dropped = dir.set_tokens(tokens).await?;
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&cfg.bind).await.with_context(|| format!("listening on {}", cfg.bind))?;
|
||||
tracing::info!(bind = %cfg.bind, tokens = cfg.tokens.len(), dropped, "directory started");
|
||||
tokio::spawn(ipx_directory::refresher(dir.clone()));
|
||||
tokio::spawn(ipx_directory::reloader(dir.clone(), path.to_owned()));
|
||||
axum::serve(listener, ipx_directory::router(dir))
|
||||
.with_graceful_shutdown(async {
|
||||
let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("SIGTERM handler");
|
||||
tokio::select! {
|
||||
_ = term.recv() => {}
|
||||
_ = tokio::signal::ctrl_c() => {}
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
518
crates/directory/src/store.rs
Normal file
518
crates/directory/src/store.rs
Normal file
@@ -0,0 +1,518 @@
|
||||
//! The directory's database: the feeds instances have sent, what each instance sent, and what
|
||||
//! an admin has blocked. SQL written by hand, as ipx's is, so it runs on SQLite (tests) and
|
||||
//! Postgres (production) both: `$1` parameters, `ON CONFLICT`, booleans as themselves.
|
||||
|
||||
use anyhow::Result;
|
||||
use sea_orm::{ConnectionTrait, DatabaseConnection, Statement, TransactionTrait, Value};
|
||||
|
||||
use crate::{Blocked, FeedPage, Item, Listing, SnapshotEntry, SnapshotResult};
|
||||
|
||||
/// A feed's items as its page shows them, and as many as ipx's own Directory page showed.
|
||||
pub const ITEMS: usize = 20;
|
||||
|
||||
/// How long a feed's failure goes unexplained: a site down for an hour is not called gone (ipx#143).
|
||||
const FLAG_AFTER_SECS: i64 = 24 * 3600;
|
||||
|
||||
pub struct Store {
|
||||
db: DatabaseConnection,
|
||||
}
|
||||
|
||||
/// What the refresher needs to ask for a feed again.
|
||||
pub struct Due {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
pub etag: Option<String>,
|
||||
pub last_modified: Option<String>,
|
||||
}
|
||||
|
||||
/// What a fetch found, to be stored.
|
||||
pub enum Refreshed {
|
||||
NotModified,
|
||||
Feed {
|
||||
parsed: ipx_feed::ParsedFeed,
|
||||
etag: Option<String>,
|
||||
last_modified: Option<String>,
|
||||
},
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
pub fn now() -> i64 {
|
||||
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() as i64)
|
||||
}
|
||||
|
||||
/// The address as every instance would write it: scheme and host lower-cased, a default port
|
||||
/// and a fragment dropped. None for anything that is not http(s).
|
||||
pub fn normalise(url: &str) -> Option<String> {
|
||||
let mut u = url::Url::parse(url.trim()).ok()?;
|
||||
if !matches!(u.scheme(), "http" | "https") || u.host_str().is_none() {
|
||||
return None;
|
||||
}
|
||||
u.set_fragment(None);
|
||||
Some(u.to_string())
|
||||
}
|
||||
|
||||
/// A feed's id: the first 64 bits of its normalised address's SHA-256, so every instance, and
|
||||
/// anyone else, arrives at the same id for the same feed without asking.
|
||||
pub fn feed_id(normalised: &str) -> String {
|
||||
use sha2::Digest;
|
||||
sha2::Sha256::digest(normalised.as_bytes())[..8].iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
const SCHEMA: &[&str] = &[
|
||||
"CREATE TABLE IF NOT EXISTS feeds (
|
||||
id TEXT PRIMARY KEY,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
title TEXT,
|
||||
image TEXT,
|
||||
category TEXT,
|
||||
subcategory TEXT,
|
||||
description TEXT,
|
||||
podcast BOOLEAN NOT NULL DEFAULT false,
|
||||
items TEXT,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
checked_at BIGINT,
|
||||
last_error TEXT,
|
||||
error_since BIGINT
|
||||
)",
|
||||
"CREATE TABLE IF NOT EXISTS snapshots (
|
||||
instance TEXT NOT NULL,
|
||||
feed_id TEXT NOT NULL,
|
||||
subscribers BIGINT NOT NULL,
|
||||
PRIMARY KEY (instance, feed_id)
|
||||
)",
|
||||
"CREATE INDEX IF NOT EXISTS snapshots_feed ON snapshots (feed_id)",
|
||||
"CREATE TABLE IF NOT EXISTS blocked (
|
||||
id TEXT PRIMARY KEY,
|
||||
url TEXT,
|
||||
title TEXT,
|
||||
at BIGINT NOT NULL
|
||||
)",
|
||||
];
|
||||
|
||||
impl Store {
|
||||
/// A database of its own for a test: a fresh schema on IPX_TEST_DATABASE_URL's Postgres when
|
||||
/// that is set, as ipx's tests do, or else SQLite in memory.
|
||||
#[cfg(test)]
|
||||
pub async fn scratch() -> Self {
|
||||
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
|
||||
let Some(url) = std::env::var("IPX_TEST_DATABASE_URL").ok().filter(|u| !u.is_empty()) else {
|
||||
return Self::open("sqlite::memory:").await.unwrap();
|
||||
};
|
||||
let (pid, n) = (std::process::id(), N.fetch_add(1, std::sync::atomic::Ordering::Relaxed));
|
||||
// A bare connection: Store::open would make the tables in the public schema, and the
|
||||
// tests running at once raced each other doing it.
|
||||
let admin = Self { db: sea_orm::Database::connect(url.as_str()).await.unwrap() };
|
||||
// Schemas an earlier run left behind: anything of ours not made by this process.
|
||||
let old = admin
|
||||
.rows(&format!(r"SELECT nspname FROM pg_namespace WHERE nspname LIKE 'ipxd\_%' AND nspname NOT LIKE 'ipxd\_{pid}\_%'"), vec![])
|
||||
.await
|
||||
.unwrap();
|
||||
for r in old {
|
||||
let name: String = r.try_get_by_index(0).unwrap();
|
||||
admin.db.execute_unprepared(&format!("DROP SCHEMA IF EXISTS {name} CASCADE")).await.unwrap();
|
||||
}
|
||||
let schema = format!("ipxd_{pid}_{n}");
|
||||
admin.db.execute_unprepared(&format!("CREATE SCHEMA {schema}")).await.unwrap();
|
||||
let sep = if url.contains('?') { '&' } else { '?' };
|
||||
Self::open(&format!("{url}{sep}options=-c%20search_path%3D{schema}%20-c%20client_min_messages%3Dwarning")).await.unwrap()
|
||||
}
|
||||
|
||||
pub async fn open(url: &str) -> Result<Self> {
|
||||
let mut url = url.to_owned();
|
||||
if url.starts_with("postgres") && !url.contains("options=") {
|
||||
// Postgres sends a notice for every CREATE ... IF NOT EXISTS on something that exists;
|
||||
// asking for none keeps them out of the log, as ipx's connections do.
|
||||
let sep = if url.contains('?') { '&' } else { '?' };
|
||||
url = format!("{url}{sep}options=-c%20client_min_messages%3Dwarning");
|
||||
}
|
||||
let mut opts = sea_orm::ConnectOptions::new(url.clone());
|
||||
opts.sqlx_logging(false);
|
||||
// An in-memory SQLite database is one per connection: a pool of them would be several.
|
||||
if url.contains(":memory:") {
|
||||
opts.max_connections(1);
|
||||
}
|
||||
let db = sea_orm::Database::connect(opts).await?;
|
||||
for sql in SCHEMA {
|
||||
db.execute_unprepared(sql).await?;
|
||||
}
|
||||
Ok(Self { db })
|
||||
}
|
||||
|
||||
fn stmt(&self, sql: &str, values: Vec<Value>) -> Statement {
|
||||
Statement::from_sql_and_values(self.db.get_database_backend(), sql, values)
|
||||
}
|
||||
|
||||
async fn exec(&self, sql: &str, values: Vec<Value>) -> Result<u64> {
|
||||
Ok(self.db.execute_raw(self.stmt(sql, values)).await?.rows_affected())
|
||||
}
|
||||
|
||||
async fn rows(&self, sql: &str, values: Vec<Value>) -> Result<Vec<sea_orm::QueryResult>> {
|
||||
Ok(self.db.query_all_raw(self.stmt(sql, values)).await?)
|
||||
}
|
||||
|
||||
pub async fn ping(&self) -> Result<()> {
|
||||
self.rows("SELECT 1", vec![]).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replaces everything `instance` sent before with `entries`. A private-looking address or a
|
||||
/// blocked feed is left out and counted; the rest are added to the feeds, new ones to be
|
||||
/// fetched by the refresher. Feeds no instance sends any more go.
|
||||
pub async fn snapshot(&self, instance: &str, entries: &[SnapshotEntry]) -> Result<SnapshotResult> {
|
||||
let blocked: std::collections::HashSet<String> =
|
||||
self.rows("SELECT id FROM blocked", vec![]).await?.iter().map(|r| r.try_get("", "id")).collect::<Result<_, _>>()?;
|
||||
let mut result = SnapshotResult::default();
|
||||
let mut keep = std::collections::BTreeMap::new();
|
||||
for e in entries {
|
||||
let Some(url) = normalise(&e.url) else {
|
||||
result.refused_invalid += 1;
|
||||
continue;
|
||||
};
|
||||
// Checked here as well as by the sender: one instance's miss must not publish
|
||||
// someone's paid feed to every other.
|
||||
if ipx_feed::url_looks_private(&url) {
|
||||
result.refused_private += 1;
|
||||
continue;
|
||||
}
|
||||
let id = feed_id(&url);
|
||||
if blocked.contains(&id) {
|
||||
result.refused_blocked += 1;
|
||||
continue;
|
||||
}
|
||||
// The same feed twice in one snapshot counts once, at its larger number.
|
||||
let n = e.subscribers.max(0);
|
||||
let slot = keep.entry(id).or_insert((url, n));
|
||||
slot.1 = slot.1.max(n);
|
||||
}
|
||||
result.accepted = keep.len();
|
||||
|
||||
let tx = self.db.begin().await?;
|
||||
let backend = tx.get_database_backend();
|
||||
let run = |sql: &str, values: Vec<Value>| Statement::from_sql_and_values(backend, sql, values);
|
||||
tx.execute_raw(run("DELETE FROM snapshots WHERE instance = $1", vec![instance.into()])).await?;
|
||||
for (id, (url, n)) in &keep {
|
||||
tx.execute_raw(run(
|
||||
"INSERT INTO feeds (id, url) VALUES ($1, $2) ON CONFLICT (id) DO NOTHING",
|
||||
vec![id.clone().into(), url.clone().into()],
|
||||
))
|
||||
.await?;
|
||||
tx.execute_raw(run(
|
||||
"INSERT INTO snapshots (instance, feed_id, subscribers) VALUES ($1, $2, $3)",
|
||||
vec![instance.into(), id.clone().into(), (*n).into()],
|
||||
))
|
||||
.await?;
|
||||
}
|
||||
tx.execute_raw(run("DELETE FROM feeds WHERE id NOT IN (SELECT feed_id FROM snapshots)", vec![])).await?;
|
||||
tx.commit().await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Drops the snapshots of every instance not in `names`: a token taken out of the config
|
||||
/// file takes its counts with it.
|
||||
pub async fn keep_instances(&self, names: &[String]) -> Result<u64> {
|
||||
let mut dropped = 0;
|
||||
let present: Vec<String> = self
|
||||
.rows("SELECT DISTINCT instance FROM snapshots", vec![])
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| r.try_get("", "instance"))
|
||||
.collect::<Result<_, _>>()?;
|
||||
for name in present.iter().filter(|n| !names.contains(n)) {
|
||||
dropped += self.drop_instance(name).await?;
|
||||
}
|
||||
Ok(dropped)
|
||||
}
|
||||
|
||||
/// Drops what one instance sent; how many feeds it had.
|
||||
pub async fn drop_instance(&self, name: &str) -> Result<u64> {
|
||||
let n = self.exec("DELETE FROM snapshots WHERE instance = $1", vec![name.into()]).await?;
|
||||
self.exec("DELETE FROM feeds WHERE id NOT IN (SELECT feed_id FROM snapshots)", vec![]).await?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
const LISTING: &str = "SELECT f.id, f.url, f.title, f.image, f.category, f.subcategory, f.podcast,
|
||||
CAST(sum(s.subscribers) AS BIGINT) AS subscribers
|
||||
FROM feeds f JOIN snapshots s ON s.feed_id = f.id";
|
||||
|
||||
fn listing(r: &sea_orm::QueryResult) -> Result<Listing> {
|
||||
Ok(Listing {
|
||||
id: r.try_get("", "id")?,
|
||||
url: r.try_get("", "url")?,
|
||||
title: r.try_get("", "title")?,
|
||||
image: r.try_get("", "image")?,
|
||||
category: r.try_get("", "category")?,
|
||||
subcategory: r.try_get("", "subcategory")?,
|
||||
podcast: r.try_get("", "podcast")?,
|
||||
subscribers: r.try_get("", "subscribers")?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Every feed, A to Z by title (one not fetched yet has none and sorts by its address).
|
||||
///
|
||||
/// ponytail: the whole list in one answer, as ipx's own Directory was; page it when it
|
||||
/// grows past a few thousand feeds.
|
||||
pub async fn list(&self) -> Result<Vec<Listing>> {
|
||||
let sql = format!("{} GROUP BY f.id ORDER BY lower(coalesce(f.title, f.url)), f.id", Self::LISTING);
|
||||
self.rows(&sql, vec![]).await?.iter().map(Self::listing).collect()
|
||||
}
|
||||
|
||||
/// The most subscribed, most first.
|
||||
pub async fn popular(&self, limit: u64) -> Result<Vec<Listing>> {
|
||||
let sql = format!(
|
||||
"{} GROUP BY f.id ORDER BY sum(s.subscribers) DESC, lower(coalesce(f.title, f.url)), f.id LIMIT $1",
|
||||
Self::LISTING
|
||||
);
|
||||
self.rows(&sql, vec![(limit as i64).into()]).await?.iter().map(Self::listing).collect()
|
||||
}
|
||||
|
||||
/// One feed with its description and latest items.
|
||||
pub async fn feed(&self, id: &str) -> Result<Option<FeedPage>> {
|
||||
let sql = format!("{} WHERE f.id = $1 GROUP BY f.id", Self::LISTING);
|
||||
let Some(r) = self.rows(&sql, vec![id.into()]).await?.into_iter().next() else { return Ok(None) };
|
||||
let listing = Self::listing(&r)?;
|
||||
let r = self
|
||||
.rows("SELECT description, items, last_error, error_since FROM feeds WHERE id = $1", vec![id.into()])
|
||||
.await?
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| anyhow::anyhow!("feed {id} went while it was read"))?;
|
||||
let items: Option<String> = r.try_get("", "items")?;
|
||||
let last_error: Option<String> = r.try_get("", "last_error")?;
|
||||
let error_since: Option<i64> = r.try_get("", "error_since")?;
|
||||
Ok(Some(FeedPage {
|
||||
listing,
|
||||
description: r.try_get("", "description")?,
|
||||
failing: last_error.map(|e| failure_words(&e, now() - error_since.unwrap_or_else(now))),
|
||||
items: items.and_then(|i| serde_json::from_str(&i).ok()).unwrap_or_default(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Removes a feed and blocks it, so no snapshot brings it back. False when it was neither
|
||||
/// listed nor blocked already.
|
||||
pub async fn block(&self, id: &str) -> Result<bool> {
|
||||
let r = self.rows("SELECT url, title FROM feeds WHERE id = $1", vec![id.into()]).await?.into_iter().next();
|
||||
let (url, title): (Option<String>, Option<String>) = match &r {
|
||||
Some(r) => (r.try_get("", "url")?, r.try_get("", "title")?),
|
||||
None => (None, None),
|
||||
};
|
||||
let added = self
|
||||
.exec(
|
||||
"INSERT INTO blocked (id, url, title, at) VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO NOTHING",
|
||||
vec![id.into(), url.into(), title.into(), now().into()],
|
||||
)
|
||||
.await?;
|
||||
self.exec("DELETE FROM snapshots WHERE feed_id = $1", vec![id.into()]).await?;
|
||||
let removed = self.exec("DELETE FROM feeds WHERE id = $1", vec![id.into()]).await?;
|
||||
Ok(added + removed > 0)
|
||||
}
|
||||
|
||||
/// Unblocks a feed; it comes back with the next snapshot that has it.
|
||||
pub async fn unblock(&self, id: &str) -> Result<bool> {
|
||||
Ok(self.exec("DELETE FROM blocked WHERE id = $1", vec![id.into()]).await? > 0)
|
||||
}
|
||||
|
||||
pub async fn blocked(&self) -> Result<Vec<Blocked>> {
|
||||
self.rows("SELECT id, url, title, at FROM blocked ORDER BY at DESC, id", vec![])
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| Ok(Blocked { id: r.try_get("", "id")?, url: r.try_get("", "url")?, title: r.try_get("", "title")?, at: r.try_get("", "at")? }))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Feeds never fetched, then those fetched longest ago, older than `every` seconds.
|
||||
pub async fn due(&self, every: i64, limit: u64) -> Result<Vec<Due>> {
|
||||
self.rows(
|
||||
"SELECT id, url, etag, last_modified FROM feeds
|
||||
WHERE checked_at IS NULL OR checked_at < $1
|
||||
ORDER BY coalesce(checked_at, 0), id LIMIT $2",
|
||||
vec![(now() - every).into(), (limit as i64).into()],
|
||||
)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| {
|
||||
Ok(Due {
|
||||
id: r.try_get("", "id")?,
|
||||
url: r.try_get("", "url")?,
|
||||
etag: r.try_get("", "etag")?,
|
||||
last_modified: r.try_get("", "last_modified")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn due_one(&self, id: &str) -> Result<Option<Due>> {
|
||||
Ok(self
|
||||
.rows("SELECT id, url, etag, last_modified FROM feeds WHERE id = $1", vec![id.into()])
|
||||
.await?
|
||||
.first()
|
||||
.map(|r| -> Result<Due> {
|
||||
Ok(Due {
|
||||
id: r.try_get("", "id")?,
|
||||
url: r.try_get("", "url")?,
|
||||
etag: r.try_get("", "etag")?,
|
||||
last_modified: r.try_get("", "last_modified")?,
|
||||
})
|
||||
})
|
||||
.transpose()?)
|
||||
}
|
||||
|
||||
pub async fn save(&self, id: &str, got: Refreshed) -> Result<()> {
|
||||
let t = now();
|
||||
match got {
|
||||
Refreshed::NotModified => {
|
||||
self.exec("UPDATE feeds SET checked_at = $1, last_error = NULL, error_since = NULL WHERE id = $2", vec![t.into(), id.into()]).await?;
|
||||
}
|
||||
Refreshed::Failed(err) => {
|
||||
self.exec(
|
||||
"UPDATE feeds SET checked_at = $1, last_error = $2, error_since = coalesce(error_since, $1) WHERE id = $3",
|
||||
vec![t.into(), err.into(), id.into()],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Refreshed::Feed { parsed, etag, last_modified } => {
|
||||
let path = parsed.category.as_deref().map(ipx_feed::category_path);
|
||||
let podcast = parsed.entries.iter().flat_map(|e| &e.enclosures).any(|e| {
|
||||
e.mime.as_deref().is_some_and(|m| m.starts_with("audio/") || m.starts_with("video/"))
|
||||
});
|
||||
let mut entries = parsed.entries;
|
||||
entries.sort_by_key(|e| std::cmp::Reverse(e.published));
|
||||
// What an item is, never its files: their addresses are where a key would be.
|
||||
let items: Vec<Item> = entries
|
||||
.into_iter()
|
||||
.take(ITEMS)
|
||||
.map(|e| Item { guid: e.guid, title: e.title, link: e.link, published: e.published, duration: e.duration, description: e.description })
|
||||
.collect();
|
||||
self.exec(
|
||||
"UPDATE feeds SET title = $1, image = $2, category = $3, subcategory = $4, description = $5,
|
||||
podcast = $6, items = $7, etag = $8, last_modified = $9, checked_at = $10,
|
||||
last_error = NULL, error_since = NULL
|
||||
WHERE id = $11",
|
||||
vec![
|
||||
parsed.title.into(),
|
||||
parsed.image.into(),
|
||||
path.as_ref().map(|p| p.0.clone()).into(),
|
||||
path.and_then(|p| p.1).into(),
|
||||
parsed.description.into(),
|
||||
podcast.into(),
|
||||
serde_json::to_string(&items)?.into(),
|
||||
etag.into(),
|
||||
last_modified.into(),
|
||||
t.into(),
|
||||
id.into(),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// A feed's last error as its page says it: explained once it has been failing a day, or else
|
||||
/// only that the check failed. Never the error itself, which is for the log.
|
||||
fn failure_words(err: &str, failing_for: i64) -> String {
|
||||
Some(err)
|
||||
.filter(|_| failing_for >= FLAG_AFTER_SECS)
|
||||
.and_then(ipx_feed::explain_failure)
|
||||
.map_or_else(|| "Its last check failed.".to_owned(), |f| f.reason.to_owned())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn e(url: &str, n: i64) -> SnapshotEntry {
|
||||
SnapshotEntry { url: url.into(), subscribers: n }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_instance_writes_one_feed_the_same_way() {
|
||||
let a = normalise("HTTPS://Example.COM:443/feed.xml#top").unwrap();
|
||||
assert_eq!(a, "https://example.com/feed.xml");
|
||||
assert_eq!(feed_id(&a), feed_id(&normalise("https://example.com/feed.xml").unwrap()));
|
||||
assert_eq!(feed_id(&a).len(), 16);
|
||||
assert!(normalise("ftp://example.com/x").is_none());
|
||||
assert!(normalise("not a url").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snapshots_replace_what_an_instance_sent_and_add_up_across_instances() {
|
||||
let s = Store::scratch().await;
|
||||
let r = s
|
||||
.snapshot("a", &[e("https://x.test/a.xml", 3), e("https://x.test/b.xml", 1), e("https://x.test/a.xml", 5)])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.accepted, 2);
|
||||
s.snapshot("b", &[e("https://x.test/a.xml", 2)]).await.unwrap();
|
||||
let pop = s.popular(10).await.unwrap();
|
||||
assert_eq!((pop[0].url.as_str(), pop[0].subscribers), ("https://x.test/a.xml", 7));
|
||||
assert_eq!(pop[1].subscribers, 1);
|
||||
|
||||
// a no longer has b.xml: nobody does, so it goes.
|
||||
s.snapshot("a", &[e("https://x.test/a.xml", 1)]).await.unwrap();
|
||||
let all = s.list().await.unwrap();
|
||||
assert_eq!(all.len(), 1);
|
||||
assert_eq!(all[0].subscribers, 3);
|
||||
|
||||
// A token taken out of the config file takes its counts.
|
||||
s.keep_instances(&["b".into()]).await.unwrap();
|
||||
assert_eq!(s.list().await.unwrap()[0].subscribers, 2);
|
||||
s.drop_instance("b").await.unwrap();
|
||||
assert!(s.list().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn private_looking_addresses_and_blocked_feeds_are_refused() {
|
||||
let s = Store::scratch().await;
|
||||
let r = s
|
||||
.snapshot(
|
||||
"a",
|
||||
&[
|
||||
e("https://feeds.supercast.com/feeds/abc123", 1),
|
||||
e("https://x.test/feed?token=secret", 1),
|
||||
e("https://user:pass@x.test/feed", 1),
|
||||
e("https://www.patreon.com/rss/show?auth=k", 1),
|
||||
e("mailto:x@y", 1),
|
||||
e("https://x.test/public.xml", 1),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!((r.accepted, r.refused_private, r.refused_invalid), (1, 4, 1));
|
||||
|
||||
let id = s.list().await.unwrap()[0].id.clone();
|
||||
assert!(s.block(&id).await.unwrap());
|
||||
assert!(s.list().await.unwrap().is_empty());
|
||||
let r = s.snapshot("a", &[e("https://x.test/public.xml", 1)]).await.unwrap();
|
||||
assert_eq!((r.accepted, r.refused_blocked), (0, 1));
|
||||
assert_eq!(s.blocked().await.unwrap()[0].url.as_deref(), Some("https://x.test/public.xml"));
|
||||
assert!(s.unblock(&id).await.unwrap());
|
||||
assert_eq!(s.snapshot("a", &[e("https://x.test/public.xml", 1)]).await.unwrap().accepted, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_fetched_feed_shows_its_metadata_and_newest_items_but_no_files() {
|
||||
let s = Store::scratch().await;
|
||||
s.snapshot("a", &[e("https://x.test/feed.xml", 1)]).await.unwrap();
|
||||
let due = s.due(3600, 10).await.unwrap();
|
||||
assert_eq!(due.len(), 1);
|
||||
let parsed = ipx_feed::parse(include_bytes!("../../../tests/data/rss2.xml")).unwrap();
|
||||
let title = parsed.title.clone();
|
||||
s.save(&due[0].id, Refreshed::Feed { parsed, etag: Some("e".into()), last_modified: None }).await.unwrap();
|
||||
assert!(s.due(3600, 10).await.unwrap().is_empty());
|
||||
let page = s.feed(&due[0].id).await.unwrap().unwrap();
|
||||
assert_eq!(page.listing.title, title);
|
||||
assert!(page.listing.podcast);
|
||||
assert!(!page.items.is_empty() && page.items.len() <= ITEMS);
|
||||
assert!(page.items.windows(2).all(|w| w[0].published >= w[1].published));
|
||||
assert!(page.failing.is_none());
|
||||
let json = serde_json::to_string(&page).unwrap();
|
||||
assert!(!json.contains(".mp3"), "an item's file must not be in its page: {json}");
|
||||
|
||||
s.save(&due[0].id, Refreshed::Failed("HTTP 404 Not Found".into())).await.unwrap();
|
||||
let page = s.feed(&due[0].id).await.unwrap().unwrap();
|
||||
assert_eq!(page.failing.as_deref(), Some("Its last check failed."));
|
||||
assert_eq!(page.listing.title, title, "a failure keeps what was found before");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user