rays/ipodderx-rs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Steps 7 and 8: torrents, OPML, and polish

Browse Source 
librqbit replaces the vendored BitTorrent 4.2.1 tree. Torrents download
in place because seeding serves the files it downloaded, so the planned
stage-then-move would have broken it. The stall budget now also covers
magnet metadata resolution, which otherwise never returns against a dead
swarm and wedged the scan.

Adds add/rm/import/export, tracing setup, systemd units and README.

A successful swarm download is unverified: no reachable peers here.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RPyeapneuXrCdojsaiXGbe
This commit is contained in:
Ray Slakinski
2026-09-09 21:02:22 +00:00
parent b12e0c46dd
commit 833c07240b
10 changed files with 651 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
src/main.rs
View File

@@ -4,8 +4,9 @@ mod download;
mod feed;
mod ipc;
mod retention;
mod torrent;
use anyhow::Result;
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use ipc::{Command as Cmd, Emitter, Event};
use std::path::PathBuf;
@@ -46,6 +47,22 @@ enum Command {
},
/// Counts of feeds, pending and downloaded enclosures
Status,
/// Subscribe to a feed
Add {
url: String,
/// Download folder name (default: the feed title)
#[arg(long)]
folder: Option<String>,
/// Only take enclosures matching these keywords
#[arg(long, value_delimiter = ',')]
keywords: Vec<String>,
},
/// Unsubscribe. Downloads and history are left alone.
Rm { feed: String },
/// Add every feed in an OPML file
Import { file: PathBuf },
/// Write subscriptions out as OPML
Export { file: PathBuf },
/// Run the scheduler and serve the control socket
Daemon,
}
@@ -56,6 +73,17 @@ struct Ctx {
db: db::Db,
client: reqwest::Client,
out: Emitter,
/// Started on first use: a BitTorrent session binds ports and starts a DHT, which is
/// rude to do for a config that has never seen a torrent.
torrents: tokio::sync::OnceCell<torrent::Torrents>,
}
impl Ctx {
async fn torrents(&self) -> Result<&torrent::Torrents> {
self.torrents
.get_or_try_init(|| torrent::Torrents::new(&self.cfg))
.await
}
}
#[tokio::main]
@@ -80,7 +108,12 @@ async fn main() -> Result<()> {
}
Command::Reap { dry_run } => Some(Cmd::Reap { dry_run: *dry_run }),
Command::Status => Some(Cmd::Status),
Command::List | Command::Daemon => None,
Command::List
| Command::Daemon
| Command::Add { .. }
| Command::Rm { .. }
| Command::Import { .. }
| Command::Export { .. } => None,
};
if let Some(cmd) = &wire_cmd
&& !cli.local
@@ -96,11 +129,18 @@ async fn main() -> Result<()> {
.user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
.build()?,
out: Emitter::terminal(),
torrents: tokio::sync::OnceCell::new(),
};
match cli.command {
Command::List => list(&ctx, &config_path),
Command::Daemon => daemon(ctx).await,
Command::Add { url, folder, keywords } => {
add(ctx, &config_path, &url, folder, keywords).await
}
Command::Rm { feed } => rm(ctx, &config_path, &feed),
Command::Import { file } => import(ctx, &config_path, &file).await,
Command::Export { file } => export(&ctx, &file),
_ => run(&ctx, wire_cmd.expect("only List and Daemon have no wire form")).await,
}
}
@@ -174,6 +214,146 @@ async fn shutdown() {
}
}
/// Subscribes to one feed, naming it from its own title.
async fn add(
mut ctx: Ctx,
config_path: &std::path::Path,
url: &str,
folder: Option<String>,
keywords: Vec<String>,
) -> Result<()> {
if let Some((id, _)) = ctx.cfg.feeds.iter().find(|(_, f)| f.url == url) {
anyhow::bail!("already subscribed as {id:?}");
}
let id = add_one(&mut ctx, url, folder, keywords).await?;
ctx.cfg.save(config_path)?;
println!("added {id}");
Ok(())
}
/// Returns the new feed id. The title needs a fetch, so a feed that cannot be reached is
/// still added -- under a slug derived from its URL -- rather than refused.
async fn add_one(
ctx: &mut Ctx,
url: &str,
folder: Option<String>,
keywords: Vec<String>,
) -> Result<String> {
let probe = config::Feed {
url: url.to_owned(),
folder: folder.clone(),
keywords: keywords.clone(),
allow_explicit: false,
auto_download: true,
max_new_per_check: None,
username: None,
password: None,
password_env: None,
};
let title = match feed::fetch(&ctx.client, &probe, None, None).await {
Ok(feed::Fetched::Body { bytes, .. }) => feed::parse(&bytes)
.ok()
.and_then(|f| f.title)
.unwrap_or_else(|| url_stem(url)),
_ => {
tracing::warn!(url, "could not read the feed; naming it from its URL");
url_stem(url)
}
};
let id = config::unique_slug(&title, &ctx.cfg.feeds);
ctx.cfg.feeds.insert(id.clone(), probe);
Ok(id)
}
/// Host plus last path segment, for naming a feed we could not read.
fn url_stem(url: &str) -> String {
url::Url::parse(url)
.ok()
.and_then(|u| u.host_str().map(str::to_owned))
.unwrap_or_else(|| url.to_owned())
}
fn rm(mut ctx: Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> {
if ctx.cfg.feeds.remove(feed).is_none() {
anyhow::bail!("no feed with id {feed:?}");
}
ctx.cfg.save(config_path)?;
// State and files stay: re-adding the feed should not re-download its back catalogue.
println!("removed {feed}; downloads and history kept");
Ok(())
}
async fn import(mut ctx: Ctx, config_path: &std::path::Path, file: &std::path::Path) -> Result<()> {
let text = std::fs::read_to_string(file)
.with_context(|| format!("reading {}", file.display()))?;
let doc = opml::OPML::from_str(&text).map_err(|e| anyhow::anyhow!("parsing OPML: {e}"))?;
let mut found = vec![];
collect_outlines(&doc.body.outlines, &mut found);
let mut added = 0;
for (title, url) in found {
if ctx.cfg.feeds.values().any(|f| f.url == url) {
continue;
}
// Name it from the OPML title rather than refetching every feed.
let id = config::unique_slug(&title, &ctx.cfg.feeds);
ctx.cfg.feeds.insert(
id.clone(),
config::Feed {
url,
folder: None,
keywords: vec![],
allow_explicit: false,
auto_download: true,
max_new_per_check: None,
username: None,
password: None,
password_env: None,
},
);
println!("added {id}");
added += 1;
}
ctx.cfg.save(config_path)?;
println!("{added} feed(s) imported");
Ok(())
}
/// OPML nests feeds inside folder outlines, so this walks the whole tree.
fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, String)>) {
for o in outlines {
if let Some(url) = &o.xml_url {
let title = o.title.clone().unwrap_or_else(|| o.text.clone());
out.push((title, url.clone()));
}
collect_outlines(&o.outlines, out);
}
}
fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> {
let mut doc = opml::OPML::default();
doc.head = Some(opml::Head {
title: Some("ipx subscriptions".into()),
..Default::default()
});
for (id, feed) in &ctx.cfg.feeds {
let title = ctx
.db
.feed_summary(id)
.ok()
.and_then(|s| s.title)
.unwrap_or_else(|| id.clone());
doc.add_feed(&title, &feed.url);
}
let xml = doc.to_string().map_err(|e| anyhow::anyhow!("writing OPML: {e}"))?;
std::fs::write(file, xml).with_context(|| format!("writing {}", file.display()))?;
println!("exported {} feed(s) to {}", ctx.cfg.feeds.len(), file.display());
Ok(())
}
fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> {
if ctx.cfg.feeds.is_empty() {
println!("No feeds configured in {}", config_path.display());
@@ -332,13 +512,37 @@ async fn scan_one(
for item in ctx.db.pending(id, budget)? {
if download::looks_like_torrent(&item.url, item.mime.as_deref()) {
// Step 7 owns these; recorded so nothing re-queues them meanwhile.
ctx.db.mark_enclosure(&item.url, "torrent", None)?;
ctx.out.emit(Event::TorrentDeferred {
feed: id.to_string(),
url: item.url.clone(),
});
scan.torrents += 1;
if !ctx.cfg.torrent.enabled {
ctx.db.mark_enclosure(&item.url, "skipped", Some("torrents disabled"))?;
ctx.out.emit(Event::TorrentDeferred {
feed: id.to_string(),
url: item.url.clone(),
});
scan.torrents += 1;
continue;
}
match torrent_one(ctx, id, &item.url, &dest_dir).await {
Ok((path, bytes)) => {
ctx.db.mark_downloaded(&item.url, &path, bytes)?;
ctx.out.emit(Event::DownloadDone {
feed: id.to_string(),
url: item.url.clone(),
path: path.display().to_string(),
bytes,
});
scan.downloaded += 1;
}
Err(e) => {
let msg = format!("{e:#}");
ctx.out.emit(Event::DownloadError {
feed: id.to_string(),
url: item.url.clone(),
msg: msg.clone(),
});
ctx.db.mark_enclosure(&item.url, "error", Some(&msg))?;
scan.failed += 1;
}
}
continue;
}
match fetch_one(ctx, id, feed_cfg, &item.url, &dest_dir).await {
@@ -417,10 +621,14 @@ async fn fetch_one(
.await?;
if matches!(got.kind, download::Kind::Torrent) {
// The MIME lied. Don't file a .torrent as an episode.
// The MIME lied. Hand the URL to the torrent session instead of filing a .torrent
// as if it were an episode.
let _ = tokio::fs::remove_file(&got.tmp).await;
ctx.db.mark_enclosure(url, "torrent", None)?;
anyhow::bail!("body is a torrent, deferred to the torrent downloader");
if !ctx.cfg.torrent.enabled {
ctx.db.mark_enclosure(url, "skipped", Some("torrents disabled"))?;
anyhow::bail!("body is a torrent and torrents are disabled");
}
return torrent_one(ctx, feed_id, url, dest_dir).await;
}
let path = download::place(&got, dest_dir).await?;
@@ -428,6 +636,36 @@ async fn fetch_one(
Ok((path, got.bytes))
}
/// Torrent progress is reported the same way an HTTP download's is, throttled to whole
/// percents so a UI is not flooded.
async fn torrent_one(
ctx: &Ctx,
feed_id: &str,
url: &str,
dest_dir: &std::path::Path,
) -> Result<(PathBuf, u64)> {
let name = download::filename_for(url, None);
let mut last_pct = -1i64;
ctx.torrents()
.await?
.fetch(&ctx.cfg, url, dest_dir, |done, total| {
if total > 0 {
let pct = (done * 100 / total) as i64;
if pct > last_pct {
last_pct = pct;
ctx.out.emit(Event::Progress {
feed: feed_id.to_string(),
url: url.to_string(),
file: name.clone(),
done,
total: Some(total),
});
}
}
})
.await
}
fn ago(t: Option<i64>) -> String {
let Some(t) = t else { return "never".into() };
format!("{} ago", duration((db::now() - t).max(0) as u64))