The directory has a robots.txt and a sitemap (#380)

Crawlers asked for both about 70 times a day and got 404. robots.txt
keeps them off /search, which is any query string at all, and the token-only
API, and names the sitemap; the sitemap lists /, /all, each category and each
feed. Both need whole addresses, so they are written for the host asked for:
directory.ipxreader.app through the tunnel, whose cloudflared says https, or
the LAN address.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/directory/CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- A `robots.txt` and a sitemap listing the home page, A to Z, every category and every feed, so search engines find the pages, not only the links between them (#380).
 
 - `/metrics` counts its requests and their times by route and status, the feeds it checks and those that failed, and the snapshots it takes (#370).
 - Its log lines inside a trace carry the trace's and span's ids, as ipx's do, so a failed feed check leads to its trace (#375).

crates/directory/src/api.rs

@@ -381,6 +381,12 @@ mod tests {
         assert_eq!(get_page(&app, "/category/Nothing").await.0, 404);
         assert_eq!(get_page(&app, "/search?q=").await.0, 303);
         assert_eq!(get_page(&app, "/site.js").await.0, 200);
+        // Crawlers are pointed at every page (#380).
+        let (s, robots, _) = get_page(&app, "/robots.txt").await;
+        assert!(s == 200 && robots.contains("Disallow: /v1/") && robots.contains("/sitemap.xml"), "{robots}");
+        let (s, map, _) = get_page(&app, "/sitemap.xml").await;
+        assert_eq!(s, 200);
+        assert!(map.contains(&format!("/feed/{id}</loc>")) && map.contains("/category/Podcasting</loc>"), "{map}");
         // Reading the pages takes no token; the API still does.
         assert_eq!(call(&app, "GET", "/v1/feeds", None, None).await.0, 401);
         assert_eq!(call(&app, "DELETE", &format!("/v1/feeds/{id}"), None, None).await.0, 401);

crates/directory/src/site.rs

@@ -25,6 +25,8 @@ pub fn router() -> Router<Shared> {
         .route("/search", get(search))
         .route("/category/{name}", get(category))
         .route("/feed/{id}", get(feed))
+        .route("/robots.txt", get(robots))
+        .route("/sitemap.xml", get(sitemap))
         .route("/site.css", get(|| asset("text/css; charset=utf-8", include_bytes!("site.css"))))
         .route("/site.js", get(|| asset("text/javascript; charset=utf-8", include_bytes!("site.js"))))
         .route("/logo.svg", get(|| asset("image/svg+xml", include_bytes!("../../../web/logo.svg"))))
@@ -40,6 +42,48 @@ async fn asset(kind: &'static str, body: &'static [u8]) -> Response {
     ([(header::CONTENT_TYPE, kind), (header::CACHE_CONTROL, "public, max-age=86400")], body).into_response()
 }
 
+/// The address this was asked for at, which both files must give whole: the public name through
+/// the tunnel, whose cloudflared says https, or the LAN address. Only a host name's characters are
+/// kept, since the header is the caller's to write.
+fn origin(h: &axum::http::HeaderMap) -> String {
+    let get = |k: &str| h.get(k).and_then(|v| v.to_str().ok()).unwrap_or("");
+    let host: String =
+        get("host").chars().filter(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | ':' | '[' | ']')).collect();
+    let scheme = if get("x-forwarded-proto") == "https" { "https" } else { "http" };
+    format!("{scheme}://{host}")
+}
+
+/// Crawlers may read every page but a search, which is any query at all, and the API, which takes
+/// a token (#380).
+async fn robots(h: axum::http::HeaderMap) -> Response {
+    let body = format!("User-agent: *\nDisallow: /search\nDisallow: /v1/\n\nSitemap: {}/sitemap.xml\n", origin(&h));
+    ([(header::CONTENT_TYPE, "text/plain; charset=utf-8"), (header::CACHE_CONTROL, "public, max-age=86400")], body)
+        .into_response()
+}
+
+/// Every page there is to find: the home page, A to Z, each category and each feed (#380).
+async fn sitemap(State(dir): State<Shared>, h: axum::http::HeaderMap) -> Response {
+    let all = match dir.pages_list().await {
+        Ok(a) => a,
+        Err(e) => return failed(e),
+    };
+    let o = origin(&h);
+    let cats: std::collections::BTreeSet<&str> = all.iter().map(|l| l.category.as_deref().unwrap_or("Other")).collect();
+    let mut b = String::from(
+        r#"<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">"#,
+    );
+    let paths = ["/".to_owned(), "/all".to_owned()]
+        .into_iter()
+        .chain(cats.iter().map(|c| format!("/category/{}", seg(c))))
+        .chain(all.iter().map(|l| format!("/feed/{}", seg(&l.id))));
+    for p in paths {
+        let _ = write!(b, "<url><loc>{}</loc></url>", esc(&format!("{o}{p}")));
+    }
+    b.push_str("</urlset>");
+    ([(header::CONTENT_TYPE, "application/xml; charset=utf-8"), (header::CACHE_CONTROL, "public, max-age=3600")], b)
+        .into_response()
+}
+
 #[derive(serde::Deserialize, Default)]
 struct Filter {
     /// `podcasts` or `blogs`; anything else is both.