From a13f19136ced743b56c5fae085c5c7a736d52ada Mon Sep 17 00:00:00 2001 From: rays Date: Wed, 9 Sep 2026 19:37:02 +0000 Subject: [PATCH] Step 2: TOML config and SQLite state config.rs replaces iPXSettings.py and feeds.plist; db.rs replaces the per-feed .ipxd plists, history.dat and qmcache.dat. enclosures.url is UNIQUE, which is the dedupe key the old pickle history provided. ipx list is the first working subcommand. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RPyeapneuXrCdojsaiXGbe --- Cargo.lock | 1 + Cargo.toml | 1 + PROGRESS.md | 32 +++++- src/config.rs | 263 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/db.rs | 156 ++++++++++++++++++++++++++++++ src/main.rs | 64 +++++++++++- 6 files changed, 512 insertions(+), 5 deletions(-) create mode 100644 src/config.rs create mode 100644 src/db.rs diff --git a/Cargo.lock b/Cargo.lock index 12988a2..acd4d94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1500,6 +1500,7 @@ version = "0.1.0" dependencies = [ "anyhow", "atom_syndication", + "chrono", "clap", "dirs", "infer", diff --git a/Cargo.toml b/Cargo.toml index 8d0c233..adbb79c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] anyhow = "1.0.104" atom_syndication = "0.12.10" +chrono = { version = "0.4.45", default-features = false, features = ["std", "clock"] } clap = { version = "4.6.6", features = ["derive"] } dirs = "7.0.0" infer = "0.22.0" diff --git a/PROGRESS.md b/PROGRESS.md index 13b1a12..c897856 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -8,8 +8,7 @@ The full design and step list live in the plan file at - [x] **1. Repo skeleton** — git init (`main`), `cargo init --name ipx`, deps pinned, LICENSE, README, this file. -- [ ] **2. `config.rs` + `db.rs`** — TOML config structs + SQLite schema. - *Done when:* `ipx list` prints feeds from a hand-written config; DB created once. +- [x] **2. `config.rs` + `db.rs`** — TOML config structs + SQLite schema. - [ ] **3. `feed.rs`** — conditional GET, RSS-then-Atom parse, persist entries. *Done when:* `ipx fetch` populates `entries`, second run 304s. - [ ] **4. `download.rs`** — filename derivation + sanitizer, streaming download, `infer` sniff, @@ -35,6 +34,33 @@ The full design and step list live in the plan file at --- +## 2026-09-09 — Step 2: config.rs + db.rs + +`src/config.rs`: serde structs for `[general]`, `[torrent]` and `[feeds.]` with defaults, `~` +expansion, `IPX_CONFIG` / `IPX_DATA_DIR` overrides, `save()` at mode 0600, `Feed::password()` +(`password_env` beats a literal `password`), `Torrent::ports()` parsing `"6881-6889"`. A missing +config file loads as an empty one so a fresh install works. Retention defaults are 0/0 +(unlimited, keep forever) — nothing gets deleted until Ray asks for it. + +`src/db.rs`: schema exactly as planned, WAL + `busy_timeout`, `Db::open()` idempotent, connection +behind a `Mutex`, `feed_summary()` for `list`, `now()` helper. `enclosures.url` is UNIQUE — the +dedupe key that replaces `history.dat`. + +`src/main.rs`: clap skeleton with `ipx list`. Only the subcommands that work exist; the rest arrive +with their steps. + +Verified: `cargo test` 5/5 green (config defaults, port-range fallback incl. reversed range, +password_env precedence, schema idempotency, enclosure-url uniqueness). Gate met — `ipx --config + list` printed both feeds and created `state.db` once across two runs. + +Deferred: nothing. Two dead-code warnings (`Config::save`, `Feed::password`) are expected; steps 3 +and 8 consume them. +Next: step 3 — `feed.rs`. + +Also added `chrono` 0.4 (std, clock) for RFC-2822 pubDate parsing in step 3. + +--- + ## 2026-09-09 — Step 1: repo skeleton Repo created at `/src/ipodderx-rs`, default branch `main`, `cargo init --name ipx` (edition 2024, @@ -62,4 +88,4 @@ Gotcha worth keeping: three feature names in the plan were wrong against current `reqwest/rustls-tls` is now `rustls`, env-var proxy support moved behind `system-proxy`, and `rss/with-syndication` does not exist. `librqbit`'s default features drag in OpenSSL; `rust-tls` is the fix. Whole tree is rustls-only now, no C TLS dependency. -Next: step 2 — `config.rs` + `db.rs`. +Next: step 2 — `config.rs` + `db.rs`. (done) diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..b5ac5d0 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,263 @@ +//! TOML configuration. Replaces iPXSettings.py and feeds.plist. + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct Config { + #[serde(default)] + pub general: General, + #[serde(default)] + pub torrent: Torrent, + /// Keyed by feed id: the TOML table name, which replaces the old genHash(feedURL). + #[serde(default)] + pub feeds: BTreeMap, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(default)] +pub struct General { + pub download_dir: PathBuf, + pub socket: PathBuf, + /// Default poll interval; a feed's own wins when it is longer. + pub interval_mins: u64, + pub organize: Organize, + /// 0 = unlimited. + pub max_total_gb: f64, + /// 0 = keep forever. + pub max_age_days: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Organize { + /// One folder per feed, named for the feed (or its `folder` override). + Feed, + /// One folder per day, MM-DD-YYYY, as iPXSettings.organizeDownloads == 1 did. + Date, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(default)] +pub struct Torrent { + pub enabled: bool, + /// Stop seeding at this ratio, or after seed_time_mins, whichever comes first. + pub seed_ratio: f64, + pub seed_time_mins: u64, + pub port_range: String, + /// Drop a torrent that has made no progress for this long. + pub stall_mins: u64, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Feed { + pub url: String, + /// Download folder name; defaults to the sanitized feed title. + #[serde(skip_serializing_if = "Option::is_none")] + pub folder: Option, + /// Every whitespace-separated word of a keyword must appear in the + /// url/title/description/categories for an enclosure to be taken. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub keywords: Vec, + #[serde(default)] + pub allow_explicit: bool, + #[serde(default = "yes")] + pub auto_download: bool, + /// Cap on new downloads per scan. None = unlimited. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_new_per_check: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub username: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password: Option, + /// Preferred over `password`: name of an env var holding the password. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub password_env: Option, +} + +fn yes() -> bool { + true +} + +impl Default for General { + fn default() -> Self { + Self { + download_dir: home().join("Podcasts"), + socket: default_socket(), + interval_mins: 60, + organize: Organize::Feed, + max_total_gb: 0.0, + max_age_days: 0, + } + } +} + +impl Default for Torrent { + fn default() -> Self { + Self { + enabled: true, + seed_ratio: 1.0, + seed_time_mins: 60, + port_range: "6881-6889".into(), + stall_mins: 30, + } + } +} + +impl Torrent { + /// Inclusive listen port range. Falls back to the BitTorrent default on garbage input. + pub fn ports(&self) -> (u16, u16) { + let parse = || -> Option<(u16, u16)> { + let (lo, hi) = self.port_range.split_once('-')?; + Some((lo.trim().parse().ok()?, hi.trim().parse().ok()?)) + }; + match parse() { + Some((lo, hi)) if lo <= hi => (lo, hi), + _ => (6881, 6889), + } + } +} + +impl Feed { + /// Resolved password: `password_env` wins over a literal `password`. + pub fn password(&self) -> Option { + if let Some(var) = &self.password_env { + return std::env::var(var).ok(); + } + self.password.clone() + } +} + +impl Config { + pub fn load(path: &Path) -> Result { + if !path.exists() { + // ponytail: a missing config is an empty one, so `list`/`add` work on a fresh install. + return Ok(Self::default()); + } + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading {}", path.display()))?; + let mut cfg: Self = + toml::from_str(&text).with_context(|| format!("parsing {}", path.display()))?; + cfg.general.download_dir = expand_tilde(&cfg.general.download_dir); + Ok(cfg) + } + + pub fn save(&self, path: &Path) -> Result<()> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir) + .with_context(|| format!("creating {}", dir.display()))?; + } + let text = toml::to_string_pretty(self)?; + std::fs::write(path, text).with_context(|| format!("writing {}", path.display()))?; + // Passwords may live in here. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + } + Ok(()) + } +} + +/// `$IPX_CONFIG`, else `$XDG_CONFIG_HOME/ipx/config.toml`. +pub fn config_path() -> PathBuf { + if let Ok(p) = std::env::var("IPX_CONFIG") { + return PathBuf::from(p); + } + dirs::config_dir() + .unwrap_or_else(|| home().join(".config")) + .join("ipx/config.toml") +} + +/// `$IPX_DATA_DIR`, else `$XDG_DATA_HOME/ipx`. +pub fn data_dir() -> PathBuf { + if let Ok(p) = std::env::var("IPX_DATA_DIR") { + return PathBuf::from(p); + } + dirs::data_dir() + .unwrap_or_else(|| home().join(".local/share")) + .join("ipx") +} + +fn default_socket() -> PathBuf { + match std::env::var_os("XDG_RUNTIME_DIR") { + Some(dir) => PathBuf::from(dir).join("ipx.sock"), + None => std::env::temp_dir().join("ipx.sock"), + } +} + +fn home() -> PathBuf { + dirs::home_dir().unwrap_or_else(|| PathBuf::from(".")) +} + +fn expand_tilde(p: &Path) -> PathBuf { + match p.strip_prefix("~") { + Ok(rest) => home().join(rest), + Err(_) => p.to_path_buf(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_a_config_and_applies_defaults() { + let cfg: Config = toml::from_str( + r#" + [general] + download_dir = "/tmp/pods" + + [feeds.example] + url = "https://example.com/feed.xml" + keywords = ["deep dive"] + "#, + ) + .unwrap(); + + assert_eq!(cfg.general.download_dir, PathBuf::from("/tmp/pods")); + assert_eq!(cfg.general.interval_mins, 60); + assert_eq!(cfg.general.organize, Organize::Feed); + assert!(cfg.torrent.enabled); + + let feed = &cfg.feeds["example"]; + assert!(feed.auto_download, "auto_download defaults on"); + assert!(!feed.allow_explicit); + assert_eq!(feed.max_new_per_check, None); + assert_eq!(feed.keywords, vec!["deep dive"]); + } + + #[test] + fn port_range_falls_back_when_malformed() { + let mut t = Torrent::default(); + assert_eq!(t.ports(), (6881, 6889)); + t.port_range = "51413-51420".into(); + assert_eq!(t.ports(), (51413, 51420)); + t.port_range = "nonsense".into(); + assert_eq!(t.ports(), (6881, 6889)); + t.port_range = "900-100".into(); + assert_eq!(t.ports(), (6881, 6889), "reversed range is not a range"); + } + + #[test] + fn password_env_wins_over_literal() { + let mut f = Feed { + url: "https://x/y".into(), + folder: None, + keywords: vec![], + allow_explicit: false, + auto_download: true, + max_new_per_check: None, + username: Some("ray".into()), + password: Some("literal".into()), + password_env: None, + }; + assert_eq!(f.password().as_deref(), Some("literal")); + + unsafe { std::env::set_var("IPX_TEST_PASS", "from-env") }; + f.password_env = Some("IPX_TEST_PASS".into()); + assert_eq!(f.password().as_deref(), Some("from-env")); + } +} diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..60f5b92 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,156 @@ +//! SQLite state. Replaces the per-feed .ipxd plists, history.dat and qmcache.dat. + +use anyhow::{Context, Result}; +use rusqlite::{Connection, OptionalExtension}; +use std::path::Path; +use std::sync::Mutex; + +/// ponytail: one global connection mutex. Writes here are tiny and rare; move to a +/// spawn_blocking pool if a large feed count ever makes it contend. +pub struct Db { + conn: Mutex, +} + +const SCHEMA: &str = " +CREATE TABLE IF NOT EXISTS feeds ( + id TEXT PRIMARY KEY, + url TEXT NOT NULL, + title TEXT, + etag TEXT, + last_modified TEXT, + last_checked INTEGER, + ttl_mins INTEGER, + last_error TEXT +); + +CREATE TABLE IF NOT EXISTS entries ( + feed_id TEXT NOT NULL, + guid TEXT NOT NULL, + title TEXT, + link TEXT, + published INTEGER, + description TEXT, + first_seen INTEGER NOT NULL, + read INTEGER NOT NULL DEFAULT 0, + flagged INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (feed_id, guid) +); + +-- url is UNIQUE: this is the dedupe key, and it subsumes the old history.dat pickle. +-- A reaped file keeps its row with path = NULL and state = 'reaped', so a purged +-- episode is never fetched a second time. +CREATE TABLE IF NOT EXISTS enclosures ( + id INTEGER PRIMARY KEY, + feed_id TEXT NOT NULL, + guid TEXT NOT NULL, + url TEXT NOT NULL UNIQUE, + mime TEXT, + length INTEGER, + path TEXT, + state TEXT NOT NULL, + bytes_done INTEGER NOT NULL DEFAULT 0, + downloaded_at INTEGER, + last_error TEXT +); + +CREATE INDEX IF NOT EXISTS enclosures_entry ON enclosures (feed_id, guid); +"; + +/// What `ipx list` shows next to each configured feed. +#[derive(Debug, Default)] +pub struct FeedSummary { + pub title: Option, + pub last_checked: Option, + pub last_error: Option, + pub entries: i64, + pub downloaded: i64, +} + +impl Db { + pub fn open(path: &Path) -> Result { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir) + .with_context(|| format!("creating {}", dir.display()))?; + } + let conn = Connection::open(path) + .with_context(|| format!("opening {}", path.display()))?; + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.pragma_update(None, "foreign_keys", "ON")?; + conn.pragma_update(None, "busy_timeout", 5000)?; + conn.execute_batch(SCHEMA).context("creating schema")?; + Ok(Self { conn: Mutex::new(conn) }) + } + + /// In-memory database, for tests. + #[cfg(test)] + pub fn memory() -> Result { + let conn = Connection::open_in_memory()?; + conn.execute_batch(SCHEMA)?; + Ok(Self { conn: Mutex::new(conn) }) + } + + pub fn feed_summary(&self, feed_id: &str) -> Result { + let conn = self.conn.lock().unwrap(); + let mut sum: FeedSummary = conn + .query_row( + "SELECT title, last_checked, last_error FROM feeds WHERE id = ?1", + [feed_id], + |r| { + Ok(FeedSummary { + title: r.get(0)?, + last_checked: r.get(1)?, + last_error: r.get(2)?, + ..Default::default() + }) + }, + ) + .optional()? + .unwrap_or_default(); + + sum.entries = conn.query_row( + "SELECT count(*) FROM entries WHERE feed_id = ?1", + [feed_id], + |r| r.get(0), + )?; + sum.downloaded = conn.query_row( + "SELECT count(*) FROM enclosures WHERE feed_id = ?1 AND path IS NOT NULL", + [feed_id], + |r| r.get(0), + )?; + Ok(sum) + } +} + +/// Unix seconds. Everything time-shaped in the DB is stored this way. +pub fn now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_is_idempotent_and_summary_handles_unknown_feeds() { + let db = Db::memory().unwrap(); + // Re-running the schema must not fail: open() does this on every start. + db.conn.lock().unwrap().execute_batch(SCHEMA).unwrap(); + + let sum = db.feed_summary("never-seen").unwrap(); + assert_eq!(sum.last_checked, None); + assert_eq!(sum.entries, 0); + assert_eq!(sum.downloaded, 0); + } + + #[test] + fn enclosure_url_is_the_dedupe_key() { + let db = Db::memory().unwrap(); + let conn = db.conn.lock().unwrap(); + let insert = "INSERT INTO enclosures (feed_id, guid, url, state) VALUES ('f', 'g', 'http://x/a.mp3', 'pending')"; + conn.execute(insert, []).unwrap(); + assert!(conn.execute(insert, []).is_err(), "duplicate url must be rejected"); + } +} diff --git a/src/main.rs b/src/main.rs index 6e6b1c7..7d53230 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,63 @@ -fn main() { - println!("ipx {}", env!("CARGO_PKG_VERSION")); +mod config; +mod db; + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command(name = "ipx", version, about = "A headless podcatcher")] +struct Cli { + /// Config file (default: $XDG_CONFIG_HOME/ipx/config.toml) + #[arg(long, global = true)] + config: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Show configured feeds and their state + List, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let config_path = cli.config.unwrap_or_else(config::config_path); + let cfg = config::Config::load(&config_path)?; + let db = db::Db::open(&config::data_dir().join("state.db"))?; + + match cli.command { + Command::List => list(&cfg, &db), + } +} + +fn list(cfg: &config::Config, db: &db::Db) -> Result<()> { + if cfg.feeds.is_empty() { + println!("No feeds configured in {}", config::config_path().display()); + return Ok(()); + } + for (id, feed) in &cfg.feeds { + let s = db.feed_summary(id)?; + let title = s.title.as_deref().unwrap_or("-"); + println!("{id} {title}"); + println!(" url {}", feed.url); + println!(" last checked {}", ago(s.last_checked)); + println!(" entries {} ({} downloaded)", s.entries, s.downloaded); + if let Some(err) = &s.last_error { + println!(" last error {err}"); + } + } + Ok(()) +} + +fn ago(t: Option) -> String { + let Some(t) = t else { return "never".into() }; + let secs = (db::now() - t).max(0); + match secs { + s if s < 90 => format!("{s}s ago"), + s if s < 5400 => format!("{}m ago", s / 60), + s if s < 172_800 => format!("{}h ago", s / 3600), + s => format!("{}d ago", s / 86_400), + } }