diff --git a/crates/directory/CHANGELOG.md b/crates/directory/CHANGELOG.md index 5e19d5d..35bf6ff 100644 --- a/crates/directory/CHANGELOG.md +++ b/crates/directory/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- A `robots.txt` and a sitemap listing the home page, A to Z, every category and every feed, so search engines find the pages, not only the links between them (#380). - `/metrics` counts its requests and their times by route and status, the feeds it checks and those that failed, and the snapshots it takes (#370). - Its log lines inside a trace carry the trace's and span's ids, as ipx's do, so a failed feed check leads to its trace (#375). diff --git a/crates/directory/src/api.rs b/crates/directory/src/api.rs index 627e98d..118a102 100644 --- a/crates/directory/src/api.rs +++ b/crates/directory/src/api.rs @@ -381,6 +381,12 @@ mod tests { assert_eq!(get_page(&app, "/category/Nothing").await.0, 404); assert_eq!(get_page(&app, "/search?q=").await.0, 303); assert_eq!(get_page(&app, "/site.js").await.0, 200); + // Crawlers are pointed at every page (#380). + let (s, robots, _) = get_page(&app, "/robots.txt").await; + assert!(s == 200 && robots.contains("Disallow: /v1/") && robots.contains("/sitemap.xml"), "{robots}"); + let (s, map, _) = get_page(&app, "/sitemap.xml").await; + assert_eq!(s, 200); + assert!(map.contains(&format!("/feed/{id}")) && map.contains("/category/Podcasting"), "{map}"); // Reading the pages takes no token; the API still does. assert_eq!(call(&app, "GET", "/v1/feeds", None, None).await.0, 401); assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), None, None).await.0, 401); diff --git a/crates/directory/src/site.rs b/crates/directory/src/site.rs index 73e459d..7f4684e 100644 --- a/crates/directory/src/site.rs +++ b/crates/directory/src/site.rs @@ -25,6 +25,8 @@ pub fn router() -> Router { .route("/search", get(search)) .route("/category/{name}", get(category)) .route("/feed/{id}", get(feed)) + .route("/robots.txt", get(robots)) + .route("/sitemap.xml", get(sitemap)) .route("/site.css", get(|| asset("text/css; charset=utf-8", include_bytes!("site.css")))) .route("/site.js", get(|| asset("text/javascript; charset=utf-8", include_bytes!("site.js")))) .route("/logo.svg", get(|| asset("image/svg+xml", include_bytes!("../../../web/logo.svg")))) @@ -40,6 +42,48 @@ async fn asset(kind: &'static str, body: &'static [u8]) -> Response { ([(header::CONTENT_TYPE, kind), (header::CACHE_CONTROL, "public, max-age=86400")], body).into_response() } +/// The address this was asked for at, which both files must give whole: the public name through +/// the tunnel, whose cloudflared says https, or the LAN address. Only a host name's characters are +/// kept, since the header is the caller's to write. +fn origin(h: &axum::http::HeaderMap) -> String { + let get = |k: &str| h.get(k).and_then(|v| v.to_str().ok()).unwrap_or(""); + let host: String = + get("host").chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | ':' | '[' | ']')).collect(); + let scheme = if get("x-forwarded-proto") == "https" { "https" } else { "http" }; + format!("{scheme}://{host}") +} + +/// Crawlers may read every page but a search, which is any query at all, and the API, which takes +/// a token (#380). +async fn robots(h: axum::http::HeaderMap) -> Response { + let body = format!("User-agent: *\nDisallow: /search\nDisallow: /v1/\n\nSitemap: {}/sitemap.xml\n", origin(&h)); + ([(header::CONTENT_TYPE, "text/plain; charset=utf-8"), (header::CACHE_CONTROL, "public, max-age=86400")], body) + .into_response() +} + +/// Every page there is to find: the home page, A to Z, each category and each feed (#380). +async fn sitemap(State(dir): State, h: axum::http::HeaderMap) -> Response { + let all = match dir.pages_list().await { + Ok(a) => a, + Err(e) => return failed(e), + }; + let o = origin(&h); + let cats: std::collections::BTreeSet<&str> = all.iter().map(|l| l.category.as_deref().unwrap_or("Other")).collect(); + let mut b = String::from( + r#""#, + ); + let paths = ["/".to_owned(), "/all".to_owned()] + .into_iter() + .chain(cats.iter().map(|c| format!("/category/{}", seg(c)))) + .chain(all.iter().map(|l| format!("/feed/{}", seg(&l.id)))); + for p in paths { + let _ = write!(b, "{}", esc(&format!("{o}{p}"))); + } + b.push_str(""); + ([(header::CONTENT_TYPE, "application/xml; charset=utf-8"), (header::CACHE_CONTROL, "public, max-age=3600")], b) + .into_response() +} + #[derive(serde::Deserialize, Default)] struct Filter { /// `podcasts` or `blogs`; anything else is both.