From 46fe0dddbf2c1d764b6d8c5734cd692ae53ac449 Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 23 Oct 2026 14:33:30 +0000 Subject: [PATCH] directory.toml's fetch_internal lets the refresher reach listed addresses (#329) 3bfd8f8 gave the refresher the guard with no exceptions, which broke the browser suite: its directory refreshes fixture feeds on 127.0.0.1, and five Directory tests waited on listings that never filled. The cargo tests passed because none of them fetches over HTTP. directory.toml takes fetch_internal as config.toml does, empty by default; the suite's lists 127.0.0.1. Production's has none. Co-Authored-By: Claude Opus 5.5 --- crates/directory/CHANGELOG.md | 2 +- crates/directory/src/config.rs | 4 ++++ crates/directory/src/lib.rs | 14 +++++++++----- crates/directory/src/main.rs | 2 +- docs/directory.md | 3 +++ tests/ui/global-setup.js | 2 ++ 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/crates/directory/CHANGELOG.md b/crates/directory/CHANGELOG.md index bc51bc7..d08bd96 100644 --- a/crates/directory/CHANGELOG.md +++ b/crates/directory/CHANGELOG.md @@ -18,7 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - HTTPS connections no longer trust certificates from emSign's distrusted intermediates, and certificate checks use constant-time comparisons: rustls-webpki, webpki-roots and ctutils are updated (#351). - The public pages read the list from the database at most once a minute, however many people ask. Each page asked for the whole catalogue, so a flood of requests with query strings of their own reached the database every time (#341). -- The directory no longer fetches feeds at internal addresses (loopback, the LAN, Docker's networks), named in a snapshot or reached by a redirect. It fetched them, and stored what answered as the feed's title and description (#329). +- The directory no longer fetches feeds at internal addresses (loopback, the LAN, Docker's networks), named in a snapshot or reached by a redirect, unless directory.toml's `fetch_internal` lists them. It fetched them, and stored what answered as the feed's title and description (#329). ## [1.3.0] - 2026-10-19 diff --git a/crates/directory/src/config.rs b/crates/directory/src/config.rs index 457526f..dcb8fbf 100644 --- a/crates/directory/src/config.rs +++ b/crates/directory/src/config.rs @@ -23,6 +23,10 @@ pub struct Config { /// left out, does neither. #[serde(default)] pub apple_country: String, + /// Internal addresses and networks the refresher may fetch feeds from all the same, as ipx's + /// `fetch_internal` (#329). Empty, the default, fetches nothing off the public internet. + #[serde(default)] + pub fetch_internal: Vec, #[serde(default)] pub tokens: Vec, } diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 017bbad..a3f054d 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -137,7 +137,11 @@ const PAGES_LIST_FOR: Duration = Duration::from_secs(60); pub type Shared = Arc; impl Directory { - pub fn new(store: Store, tokens: Vec) -> Result { + pub fn new( + store: Store, + tokens: Vec, + fetch_internal: ipx_feed::guard::Allowed, + ) -> Result { // No redirects: ipx_feed::fetch follows them itself, so it sees each hop. let client = reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(10)) @@ -145,8 +149,8 @@ impl Directory { .redirect(reqwest::redirect::Policy::none()) .user_agent(concat!("ipx-directory/", env!("CARGO_PKG_VERSION"))) // The feeds are named by whoever holds a write token, and redirect where their - // publishers like; none is on this network, so nothing here is fetched (#329). - .connector_layer(ipx_feed::guard::Guard::new(Default::default())) + // publishers like: nothing on this network is fetched unless listed (#329). + .connector_layer(ipx_feed::guard::Guard::new(fetch_internal)) .build()?; Ok(Arc::new(Self { store, @@ -522,7 +526,7 @@ mod tests { sha256: config::hash(&format!("secret-{n}")), }) .collect(); - let dir = Directory::new(Store::scratch().await, tokens).unwrap(); + let dir = Directory::new(Store::scratch().await, tokens, Default::default()).unwrap(); (router(dir.clone()), dir) } @@ -685,7 +689,7 @@ mod tests { #[tokio::test] async fn a_revoked_token_loses_its_access_and_its_counts() { - let dir = Directory::new(Store::scratch().await, vec![]).unwrap(); + let dir = Directory::new(Store::scratch().await, vec![], Default::default()).unwrap(); let t = |n: &str| Token { name: n.into(), access: Access::Rw, sha256: config::hash(&format!("secret-{n}")) }; dir.set_tokens(vec![t("a"), t("b")]).await.unwrap(); let app = router(dir.clone()); diff --git a/crates/directory/src/main.rs b/crates/directory/src/main.rs index 1d8fc09..2880f2c 100644 --- a/crates/directory/src/main.rs +++ b/crates/directory/src/main.rs @@ -102,7 +102,7 @@ async fn serve(path: &std::path::Path) -> Result<()> { .filter(|u| !u.trim().is_empty()) .with_context(|| format!("{} is not set: the directory needs a database URL", cfg.database_url_env))?; let store = Store::open(&url, true).await.context("opening the database")?; - let dir = Directory::new(store, vec![])?; + let dir = Directory::new(store, vec![], ipx_feed::guard::Allowed::parse(&cfg.fetch_internal)?)?; let dropped = dir.set_tokens(tokens).await?; let listener = diff --git a/docs/directory.md b/docs/directory.md index 5dd8a09..ade62fd 100644 --- a/docs/directory.md +++ b/docs/directory.md @@ -83,6 +83,9 @@ bind = "0.0.0.0:8100" metrics_bind = "0.0.0.0:9188" # Apple Podcasts' top charts for this country, weekly (see below). Empty, or left out: none. apple_country = "ca" +# Internal addresses and networks feeds may be fetched from, as ["10.0.0.5", "192.168.1.0/24"]. +# Empty, or left out: nothing off the public internet is fetched (#329). +fetch_internal = [] # The environment variable holding the database URL: postgres://... or sqlite://... database_url_env = "DIRECTORY_DATABASE_URL" diff --git a/tests/ui/global-setup.js b/tests/ui/global-setup.js index 0da3311..e83dfc4 100644 --- a/tests/ui/global-setup.js +++ b/tests/ui/global-setup.js @@ -77,6 +77,8 @@ auto_download = false fs.writeFileSync(path.join(root, 'directory.toml'), ` bind = "127.0.0.1:8795" database_url_env = "DIRECTORY_DATABASE_URL" +# The fixture feeds are on loopback, which the refresher refuses otherwise (#329). +fetch_internal = ["127.0.0.1"] [[tokens]] name = "test"