diff --git a/crates/directory/src/api.rs b/crates/directory/src/api.rs index 0036f59..c134c69 100644 --- a/crates/directory/src/api.rs +++ b/crates/directory/src/api.rs @@ -433,6 +433,26 @@ mod tests { } } + #[tokio::test] + async fn dropping_a_servers_snapshot_takes_only_the_feeds_nobody_else_lists() { + let (app, dir) = app().await; + let feed = |url: &str| serde_json::json!({ "url": url, "subscribers": 1 }); + let mine = serde_json::json!([feed("https://only-rw.test/feed"), feed("https://both.test/feed")]); + assert_eq!(call(&app, "POST", "/v1/snapshot", Some("rw"), Some(mine)).await.0, 200); + let theirs = serde_json::json!([feed("https://both.test/feed")]); + assert_eq!(call(&app, "POST", "/v1/snapshot", Some("rwa"), Some(theirs)).await.0, 200); + + assert_eq!(call(&app, "DELETE", "/v1/instances/rw-token", Some("rw"), None).await.0, 403, "admins only"); + assert_eq!(call(&app, "DELETE", "/v1/instances/rw-token", Some("rwa"), None).await.0, 204); + let left: Vec<(String, i64)> = + dir.store.instances().await.unwrap().into_iter().map(|i| (i.name, i.feeds)).collect(); + assert_eq!(left, [("rwa-token".to_owned(), 1)]); + let gone = dir.store.due_one(&store::feed_id("https://only-rw.test/feed")).await.unwrap(); + assert!(gone.is_none(), "nobody lists it now"); + let kept = dir.store.due_one(&store::feed_id("https://both.test/feed")).await.unwrap(); + assert!(kept.is_some(), "another server still does"); + } + #[tokio::test] async fn each_access_level_does_what_it_may_and_no_more() { let (app, dir) = app().await; diff --git a/crates/directory/src/apple.rs b/crates/directory/src/apple.rs index 29b9a56..53efcf7 100644 --- a/crates/directory/src/apple.rs +++ b/crates/directory/src/apple.rs @@ -23,8 +23,10 @@ const RETRY: Duration = Duration::from_secs(3600); /// The top this many overall, and of each category: about a thousand shows once they overlap. const TOP: u32 = 100; const TOP_OF_EACH: u32 = 50; -/// Apple asks for no more than about twenty calls a minute. -const PAUSE: Duration = Duration::from_secs(3); +/// Apple asks for no more than about twenty calls a minute. A test's stand-in asks for none. +const PAUSE: Duration = if cfg!(test) { Duration::ZERO } else { Duration::from_secs(3) }; +/// Where Apple's charts and lookup API are. +const APPLE: &str = "https://itunes.apple.com"; /// Syncs when a week has passed since the last, forever. pub async fn run(dir: Shared, country: String) { @@ -46,7 +48,7 @@ pub async fn run(dir: Shared, country: String) { continue; } let span = tracing::info_span!("apple_sync", country = %country); - match sync(&dir, &client, &country).instrument(span).await { + match sync(&dir, &client, APPLE, &country).instrument(span).await { Ok(()) => { let _ = dir.store.set_synced(INSTANCE, crate::store::now()).await; } @@ -58,8 +60,8 @@ pub async fn run(dir: Shared, country: String) { } } -async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<()> { - let genres: Genres = get(client, "https://itunes.apple.com/WebObjects/MZStoreServices.woa/ws/genres?id=26").await?; +async fn sync(dir: &Shared, client: &reqwest::Client, apple: &str, country: &str) -> Result<()> { + let genres: Genres = get(client, &format!("{apple}/WebObjects/MZStoreServices.woa/ws/genres?id=26")).await?; let theirs = categories(&genres); let (added, gone) = differ(&theirs, ipx_feed::categories()); if !added.is_empty() || !gone.is_empty() { @@ -74,8 +76,8 @@ async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<( let charts = std::iter::once((None, TOP)).chain(genre_ids(&genres).into_iter().map(|g| (Some(g), TOP_OF_EACH))); for (genre, n) in charts { let url = match &genre { - Some(g) => format!("https://itunes.apple.com/{country}/rss/toppodcasts/limit={n}/genre={g}/json"), - None => format!("https://itunes.apple.com/{country}/rss/toppodcasts/limit={n}/json"), + Some(g) => format!("{apple}/{country}/rss/toppodcasts/limit={n}/genre={g}/json"), + None => format!("{apple}/{country}/rss/toppodcasts/limit={n}/json"), }; match get::(client, &url).await { Ok(chart) => ids.extend(chart_ids(&chart)), @@ -90,7 +92,7 @@ async fn sync(dir: &Shared, client: &reqwest::Client, country: &str) -> Result<( let mut entries = Vec::new(); for batch in ids.chunks(100) { - let url = format!("https://itunes.apple.com/lookup?entity=podcast&id={}", batch.join(",")); + let url = format!("{apple}/lookup?entity=podcast&id={}", batch.join(",")); let found: Lookup = get(client, &url).await?; entries.extend(lookup_entries(&found)); tokio::time::sleep(PAUSE).await; @@ -233,6 +235,72 @@ fn lookup_entries(l: &Lookup) -> Vec { mod tests { use super::*; + /// A stand-in for Apple: one category, Arts (1301); the top chart is shows 1 and 2, Arts' is 2 + /// and 3; the lookup gives 1 a public feed, 2 one with a key in it, 3 another public one. With + /// `lookup_fails`, the lookup is a 503. + async fn apple(lookup_fails: bool) -> String { + use axum::extract::{Path, Query}; + let chart = |ids: &[&str]| { + let entry: Vec<_> = + ids.iter().map(|id| serde_json::json!({ "id": { "attributes": { "im:id": id } } })).collect(); + serde_json::json!({ "feed": { "entry": entry } }) + }; + let (top, arts) = (chart(&["1", "2"]), chart(&["2", "3"])); + let app = axum::Router::new() + .route( + "/WebObjects/MZStoreServices.woa/ws/genres", + axum::routing::get(|| async { + axum::Json(serde_json::json!({ "26": { "name": "Podcasts", "subgenres": { "1301": { "name": "Arts" } } } })) + }), + ) + .route( + "/us/rss/toppodcasts/{*chart}", + axum::routing::get(move |Path(c): Path| { + let got = if c.contains("genre=1301") { arts.clone() } else { top.clone() }; + async move { axum::Json(got) } + }), + ) + .route( + "/lookup", + axum::routing::get(move |Query(q): Query>| async move { + if lookup_fails { + return Err(axum::http::StatusCode::SERVICE_UNAVAILABLE); + } + assert_eq!(q["id"], "1,2,3", "each show once, in chart order"); + Ok(axum::Json(serde_json::json!({ "results": [ + { "feedUrl": "https://one.test/feed", "primaryGenreName": "Arts" }, + { "feedUrl": "https://two.test/feed?auth=0123456789abcdef0123456789abcdef", "primaryGenreName": "Arts" }, + { "feedUrl": "https://three.test/feed", "primaryGenreName": "Podcasts" }, + ]}))) + }), + ); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let at = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await }); + format!("http://{at}") + } + + async fn apples(dir: &Shared) -> i64 { + dir.store.instances().await.unwrap().iter().find(|i| i.name == INSTANCE).map_or(0, |i| i.feeds) + } + + #[tokio::test] + async fn a_sync_takes_apples_charts_as_a_snapshot_and_a_failed_one_keeps_the_last() { + let dir = crate::Directory::new(crate::store::Store::scratch().await, vec![], Default::default()).unwrap(); + let client = reqwest::Client::new(); + sync(&dir, &client, &apple(false).await, "us").await.unwrap(); + assert_eq!(apples(&dir).await, 2, "the feed with a key in it is refused as private"); + + // An admin blocks one; the next sync leaves it out. + dir.store.block(&crate::store::feed_id("https://three.test/feed")).await.unwrap(); + sync(&dir, &client, &apple(false).await, "us").await.unwrap(); + assert_eq!(apples(&dir).await, 1); + + // Apple failing is no reason to forget last week's charts. + assert!(sync(&dir, &client, &apple(true).await, "us").await.is_err()); + assert_eq!(apples(&dir).await, 1, "the last good snapshot stands"); + } + #[test] fn apples_categories_are_read_and_compared_with_ipxs() { let genres: Genres = serde_json::from_str( diff --git a/crates/directory/src/metrics.rs b/crates/directory/src/metrics.rs index 653096d..1bdbdf6 100644 --- a/crates/directory/src/metrics.rs +++ b/crates/directory/src/metrics.rs @@ -202,6 +202,27 @@ mod tests { assert_eq!(label("a \"b\"\\"), "a \\\"b\\\"\\\\"); } + #[tokio::test] + async fn every_directory_metric_grafana_queries_is_served() { + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let at = l.local_addr().unwrap(); + tokio::spawn(serve(l)); + let text = reqwest::get(format!("http://{at}/metrics")).await.unwrap().text().await.unwrap(); + let src = [include_str!("../../../grafana/dashboard.py"), include_str!("../../../grafana/alerts.py")].concat(); + let names: std::collections::BTreeSet<&str> = src + .match_indices("directory_") + .filter(|(i, _)| !src[..*i].ends_with(|c: char| c.is_ascii_alphanumeric() || c == '_')) + .map(|(i, _)| { + let rest = &src[i..]; + &rest[..rest.find(|c: char| !(c.is_ascii_lowercase() || c == '_')).unwrap_or(rest.len())] + }) + .collect(); + assert!(names.len() >= 4, "{names:?}"); + for n in names { + assert!(text.contains(&format!("# TYPE {n} ")), "{n} is queried by grafana/ but not served: {text}"); + } + } + #[test] fn metrics_say_memory_and_feeds() { let text = super::render(std::time::Instant::now());