rays/ipx
1
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Tests for the directory's Apple sync, metrics and dropped snapshots (#402)

Browse Source 
The weekly Apple Podcasts sync runs unattended and ran in no test. Its
Apple address is now an argument (run passes Apple's), and its pause
between calls is none under test, so a test runs it against a stand-in
with one category: the top chart and the category's are taken each
show once in order, looked up, and filed as the apple-podcasts
snapshot with a feed that has a key in it refused as private; a feed an
admin blocked is left out of the next; and a sync whose lookup fails
is an error that leaves the last good snapshot as it was.

/metrics serves every directory_ metric the dashboard and alerts
query, read out of them. Dropping a server's snapshot through the API
is for admins only, and takes the feeds only it listed, keeping one
another server lists too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ray Slakinski
2026-11-07 04:32:28 +00:00
parent c00d5383de
commit 8e8307b569
3 changed files with 117 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
crates/directory/src/api.rs
View File

@@ -433,6 +433,26 @@ mod tests {
}
}
#[tokio::test]
async fn dropping_a_servers_snapshot_takes_only_the_feeds_nobody_else_lists() {
let (app, dir) = app().await;
let feed = |url: &str| serde_json::json!({ "url": url, "subscribers": 1 });
let mine = serde_json::json!([feed("https://only-rw.test/feed"), feed("https://both.test/feed")]);
assert_eq!(call(&app, "POST", "/v1/snapshot", Some("rw"), Some(mine)).await.0, 200);
let theirs = serde_json::json!([feed("https://both.test/feed")]);
assert_eq!(call(&app, "POST", "/v1/snapshot", Some("rwa"), Some(theirs)).await.0, 200);
assert_eq!(call(&app, "DELETE", "/v1/instances/rw-token", Some("rw"), None).await.0, 403, "admins only");
assert_eq!(call(&app, "DELETE", "/v1/instances/rw-token", Some("rwa"), None).await.0, 204);
let left: Vec<(String, i64)> =
dir.store.instances().await.unwrap().into_iter().map(|i| (i.name, i.feeds)).collect();
assert_eq!(left, [("rwa-token".to_owned(), 1)]);
let gone = dir.store.due_one(&store::feed_id("https://only-rw.test/feed")).await.unwrap();
assert!(gone.is_none(), "nobody lists it now");
let kept = dir.store.due_one(&store::feed_id("https://both.test/feed")).await.unwrap();
assert!(kept.is_some(), "another server still does");
}
#[tokio::test]
async fn each_access_level_does_what_it_may_and_no_more() {
let (app, dir) = app().await;

84
crates/directory/src/apple.rs
View File

@@ -23,8 +23,10 @@ const RETRY: Duration = Duration::from_secs(3600);
/// The top this many overall, and of each category: about a thousand shows once they overlap.
const TOP: u32 = 100;
const TOP_OF_EACH: u32 = 50;
/// Apple asks for no more than about twenty calls a minute.
const PAUSE: Duration = Duration::from_secs(3);
/// Apple asks for no more than about twenty calls a minute. A test's stand-in asks for none.
const PAUSE: Duration = if cfg!(test) { Duration::ZERO } else { Duration::from_secs(3) };
/// Where Apple's charts and lookup API are.
const APPLE: &str = "https://itunes.apple.com";
/// Syncs when a week has passed since the last, forever.
pub async fn run(dir: Shared, country: String) {
@@ -46,7 +48,7 @@ pub async fn run(dir: Shared, country: String) {
continue;
}
let span = tracing::info_span!("apple_sync", country = %country);
match sync(&dir, &client, &country).instrument(span).await {
match sync(&dir, &client, APPLE, &country).instrument(span).await {
Ok(()) => {
let _ = dir.store.set_synced(INSTANCE, crate::store::now()).await;
}
@@ -58,8 +60,8 @@ pub async fn run(dir: Shared, country: String) {
}
}
async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<()> {
let genres: Genres = get(client, "https://itunes.apple.com/WebObjects/MZStoreServices.woa/ws/genres?id=26").await?;
async fn sync(dir: &Shared, client: &reqwest::Client, apple: &str, country: &str) -> Result<()> {
let genres: Genres = get(client, &format!("{apple}/WebObjects/MZStoreServices.woa/ws/genres?id=26")).await?;
let theirs = categories(&genres);
let (added, gone) = differ(&theirs, ipx_feed::categories());
if !added.is_empty() || !gone.is_empty() {
@@ -74,8 +76,8 @@ async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<(
let charts = std::iter::once((None, TOP)).chain(genre_ids(&genres).into_iter().map(|g| (Some(g), TOP_OF_EACH)));
for (genre, n) in charts {
let url = match &genre {
Some(g) => format!("https://itunes.apple.com/{country}/rss/toppodcasts/limit={n}/genre={g}/json"),
None => format!("https://itunes.apple.com/{country}/rss/toppodcasts/limit={n}/json"),
Some(g) => format!("{apple}/{country}/rss/toppodcasts/limit={n}/genre={g}/json"),
None => format!("{apple}/{country}/rss/toppodcasts/limit={n}/json"),
};
match get::<Chart>(client, &url).await {
Ok(chart) => ids.extend(chart_ids(&chart)),
@@ -90,7 +92,7 @@ async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<(
let mut entries = Vec::new();
for batch in ids.chunks(100) {
let url = format!("https://itunes.apple.com/lookup?entity=podcast&id={}", batch.join(","));
let url = format!("{apple}/lookup?entity=podcast&id={}", batch.join(","));
let found: Lookup = get(client, &url).await?;
entries.extend(lookup_entries(&found));
tokio::time::sleep(PAUSE).await;
@@ -233,6 +235,72 @@ fn lookup_entries(l: &Lookup) -> Vec<SnapshotEntry> {
mod tests {
use super::*;
/// A stand-in for Apple: one category, Arts (1301); the top chart is shows 1 and 2, Arts' is 2
/// and 3; the lookup gives 1 a public feed, 2 one with a key in it, 3 another public one. With
/// `lookup_fails`, the lookup is a 503.
async fn apple(lookup_fails: bool) -> String {
use axum::extract::{Path, Query};
let chart = |ids: &[&str]| {
let entry: Vec<_> =
ids.iter().map(|id| serde_json::json!({ "id": { "attributes": { "im:id": id } } })).collect();
serde_json::json!({ "feed": { "entry": entry } })
};
let (top, arts) = (chart(&["1", "2"]), chart(&["2", "3"]));
let app = axum::Router::new()
.route(
"/WebObjects/MZStoreServices.woa/ws/genres",
axum::routing::get(|| async {
axum::Json(serde_json::json!({ "26": { "name": "Podcasts", "subgenres": { "1301": { "name": "Arts" } } } }))
}),
)
.route(
"/us/rss/toppodcasts/{*chart}",
axum::routing::get(move |Path(c): Path<String>| {
let got = if c.contains("genre=1301") { arts.clone() } else { top.clone() };
async move { axum::Json(got) }
}),
)
.route(
"/lookup",
axum::routing::get(move |Query(q): Query<BTreeMap<String, String>>| async move {
if lookup_fails {
return Err(axum::http::StatusCode::SERVICE_UNAVAILABLE);
}
assert_eq!(q["id"], "1,2,3", "each show once, in chart order");
Ok(axum::Json(serde_json::json!({ "results": [
{ "feedUrl": "https://one.test/feed", "primaryGenreName": "Arts" },
{ "feedUrl": "https://two.test/feed?auth=0123456789abcdef0123456789abcdef", "primaryGenreName": "Arts" },
{ "feedUrl": "https://three.test/feed", "primaryGenreName": "Podcasts" },
]})))
}),
);
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let at = l.local_addr().unwrap();
tokio::spawn(async move { axum::serve(l, app).await });
format!("http://{at}")
}
async fn apples(dir: &Shared) -> i64 {
dir.store.instances().await.unwrap().iter().find(|i| i.name == INSTANCE).map_or(0, |i| i.feeds)
}
#[tokio::test]
async fn a_sync_takes_apples_charts_as_a_snapshot_and_a_failed_one_keeps_the_last() {
let dir = crate::Directory::new(crate::store::Store::scratch().await, vec![], Default::default()).unwrap();
let client = reqwest::Client::new();
sync(&dir, &client, &apple(false).await, "us").await.unwrap();
assert_eq!(apples(&dir).await, 2, "the feed with a key in it is refused as private");
// An admin blocks one; the next sync leaves it out.
dir.store.block(&crate::store::feed_id("https://three.test/feed")).await.unwrap();
sync(&dir, &client, &apple(false).await, "us").await.unwrap();
assert_eq!(apples(&dir).await, 1);
// Apple failing is no reason to forget last week's charts.
assert!(sync(&dir, &client, &apple(true).await, "us").await.is_err());
assert_eq!(apples(&dir).await, 1, "the last good snapshot stands");
}
#[test]
fn apples_categories_are_read_and_compared_with_ipxs() {
let genres: Genres = serde_json::from_str(

21
crates/directory/src/metrics.rs
View File

@@ -202,6 +202,27 @@ mod tests {
assert_eq!(label("a \"b\"\\"), "a \\\"b\\\"\\\\");
}
#[tokio::test]
async fn every_directory_metric_grafana_queries_is_served() {
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let at = l.local_addr().unwrap();
tokio::spawn(serve(l));
let text = reqwest::get(format!("http://{at}/metrics")).await.unwrap().text().await.unwrap();
let src = [include_str!("../../../grafana/dashboard.py"), include_str!("../../../grafana/alerts.py")].concat();
let names: std::collections::BTreeSet<&str> = src
.match_indices("directory_")
.filter(|(i, _)| !src[..*i].ends_with(|c: char| c.is_ascii_alphanumeric() || c == '_'))
.map(|(i, _)| {
let rest = &src[i..];
&rest[..rest.find(|c: char| !(c.is_ascii_lowercase() || c == '_')).unwrap_or(rest.len())]
})
.collect();
assert!(names.len() >= 4, "{names:?}");
for n in names {
assert!(text.contains(&format!("# TYPE {n} ")), "{n} is queried by grafana/ but not served: {text}");
}
}
#[test]
fn metrics_say_memory_and_feeds() {
let text = super::render(std::time::Instant::now());