Step 6: control socket and daemon
Unix-socket JSON-lines protocol replacing the printMSG sentinels, with an Emitter so scan code is agnostic about whether a terminal or a UI is watching. Daemon serves clients and runs a TTL-aware scheduler; commands funnel through one worker so scans cannot overlap, and the CLI proxies to a running daemon rather than competing with it. The pre-fetch retention sweep was emitting the terminal ReapDone event, which would have ended a client's read before the scan began. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RPyeapneuXrCdojsaiXGbe
This commit is contained in:
11
src/db.rs
11
src/db.rs
@@ -385,6 +385,17 @@ impl Db {
|
||||
}
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// (pending, downloaded) across all feeds, for the status command.
|
||||
pub fn counts(&self) -> Result<(i64, i64)> {
|
||||
let conn = self.conn.lock().unwrap();
|
||||
Ok((
|
||||
conn.query_row("SELECT count(*) FROM enclosures WHERE state = 'pending'", [], |r| r.get(0))?,
|
||||
conn.query_row("SELECT count(*) FROM enclosures WHERE path IS NOT NULL", [], |r| r.get(0))?,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Unix seconds. Everything time-shaped in the DB is stored this way.
|
||||
pub fn now() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
|
||||
277
src/ipc.rs
Normal file
277
src/ipc.rs
Normal file
@@ -0,0 +1,277 @@
|
||||
//! Unix-socket control and event stream. Replaces printMSG's `;;1;;1;;100.00;;42.31`.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tokio::sync::{broadcast, mpsc};
|
||||
|
||||
/// One JSON object per line, `ev` naming the variant.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "ev", rename_all = "snake_case")]
|
||||
pub enum Event {
|
||||
FeedStart { feed: String },
|
||||
FeedSkip { feed: String, reason: String },
|
||||
FeedDone { feed: String, new: usize, downloaded: usize, failed: usize, torrents: usize },
|
||||
FeedError { feed: String, msg: String },
|
||||
Progress {
|
||||
feed: String,
|
||||
url: String,
|
||||
file: String,
|
||||
done: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
total: Option<u64>,
|
||||
},
|
||||
DownloadDone { feed: String, url: String, path: String, bytes: u64 },
|
||||
DownloadError { feed: String, url: String, msg: String },
|
||||
TorrentDeferred { feed: String, url: String },
|
||||
Reaped { path: String, bytes: u64 },
|
||||
/// Terminal: a client that asked for work stops reading here.
|
||||
ScanDone { feeds: usize },
|
||||
ReapDone { files: usize, bytes: u64 },
|
||||
Status { feeds: usize, pending: i64, downloaded: i64 },
|
||||
Error { msg: String },
|
||||
}
|
||||
|
||||
impl Event {
|
||||
pub fn is_terminal(&self) -> bool {
|
||||
matches!(self, Event::ScanDone { .. } | Event::ReapDone { .. } | Event::Status { .. })
|
||||
}
|
||||
|
||||
/// The human rendering, for a terminal rather than a UI.
|
||||
pub fn human(&self) -> Option<String> {
|
||||
Some(match self {
|
||||
Event::FeedSkip { feed, reason } => format!("{feed}: {reason}"),
|
||||
Event::FeedDone { feed, new, downloaded, failed, torrents } => format!(
|
||||
"{feed}: {new} new entries, {downloaded} downloaded, {failed} failed, {torrents} torrents deferred"
|
||||
),
|
||||
Event::FeedError { feed, msg } => format!("{feed}: error: {msg}"),
|
||||
Event::Progress { file, done, total, .. } => match total {
|
||||
Some(t) if *t > 0 => format!(
|
||||
" {file}: {:.1}% ({:.1}/{:.1} MB)",
|
||||
*done as f64 / *t as f64 * 100.0,
|
||||
*done as f64 / 1_048_576.0,
|
||||
*t as f64 / 1_048_576.0
|
||||
),
|
||||
_ => format!(" {file}: {:.1} MB", *done as f64 / 1_048_576.0),
|
||||
},
|
||||
Event::DownloadDone { path, .. } => format!(" saved {path}"),
|
||||
Event::DownloadError { url, msg, .. } => format!(" failed {url}: {msg}"),
|
||||
Event::Reaped { path, bytes } => {
|
||||
format!("reap {path} ({:.1} MB)", *bytes as f64 / 1_048_576.0)
|
||||
}
|
||||
Event::ReapDone { files, bytes } => format!(
|
||||
"reaped {files} file(s), {:.1} MB",
|
||||
*bytes as f64 / 1_048_576.0
|
||||
),
|
||||
Event::Status { feeds, pending, downloaded } => {
|
||||
format!("{feeds} feeds, {pending} pending, {downloaded} downloaded")
|
||||
}
|
||||
Event::Error { msg } => format!("error: {msg}"),
|
||||
// Noise in a terminal; a UI still gets them on the socket.
|
||||
Event::FeedStart { .. } | Event::TorrentDeferred { .. } | Event::ScanDone { .. } => {
|
||||
return None;
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "cmd", rename_all = "snake_case")]
|
||||
pub enum Command {
|
||||
Fetch {
|
||||
#[serde(default)]
|
||||
feed: Option<String>,
|
||||
#[serde(default)]
|
||||
force: bool,
|
||||
},
|
||||
Reap {
|
||||
#[serde(default)]
|
||||
dry_run: bool,
|
||||
},
|
||||
Status,
|
||||
}
|
||||
|
||||
/// Where events go: the socket, the terminal, or both.
|
||||
#[derive(Clone)]
|
||||
pub struct Emitter {
|
||||
tx: Option<broadcast::Sender<Event>>,
|
||||
print: bool,
|
||||
}
|
||||
|
||||
impl Emitter {
|
||||
pub fn terminal() -> Self {
|
||||
Self { tx: None, print: true }
|
||||
}
|
||||
|
||||
pub fn socket(tx: broadcast::Sender<Event>, print: bool) -> Self {
|
||||
Self { tx: Some(tx), print }
|
||||
}
|
||||
|
||||
pub fn emit(&self, e: Event) {
|
||||
if let Some(tx) = &self.tx {
|
||||
// An error here only means nobody is listening yet.
|
||||
let _ = tx.send(e.clone());
|
||||
}
|
||||
if self.print && let Some(line) = e.human() {
|
||||
println!("{line}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// True when something is already listening -- i.e. a daemon owns this socket.
|
||||
pub async fn daemon_is_live(path: &Path) -> bool {
|
||||
UnixStream::connect(path).await.is_ok()
|
||||
}
|
||||
|
||||
/// Accepts connections, feeding commands to `cmds` and events from `events` back out.
|
||||
pub async fn serve(
|
||||
path: PathBuf,
|
||||
events: broadcast::Sender<Event>,
|
||||
cmds: mpsc::Sender<Command>,
|
||||
) -> Result<()> {
|
||||
// A socket file left by a crashed daemon would block the bind; a live one was already
|
||||
// rejected by the caller's daemon_is_live() check.
|
||||
if path.exists() {
|
||||
std::fs::remove_file(&path)
|
||||
.with_context(|| format!("removing stale socket {}", path.display()))?;
|
||||
}
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
}
|
||||
let listener = UnixListener::bind(&path)
|
||||
.with_context(|| format!("binding {}", path.display()))?;
|
||||
tracing::info!(socket = %path.display(), "listening");
|
||||
|
||||
loop {
|
||||
let (stream, _) = listener.accept().await?;
|
||||
let rx = events.subscribe();
|
||||
let cmds = cmds.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle(stream, rx, cmds).await {
|
||||
tracing::debug!(error = %e, "client gone");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
stream: UnixStream,
|
||||
mut rx: broadcast::Receiver<Event>,
|
||||
cmds: mpsc::Sender<Command>,
|
||||
) -> Result<()> {
|
||||
let (read, mut write) = stream.into_split();
|
||||
|
||||
// Events out.
|
||||
let writer = tokio::spawn(async move {
|
||||
while let Ok(ev) = rx.recv().await {
|
||||
let mut line = serde_json::to_string(&ev).unwrap_or_default();
|
||||
line.push('\n');
|
||||
if write.write_all(line.as_bytes()).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Commands in.
|
||||
let mut lines = BufReader::new(read).lines();
|
||||
while let Some(line) = lines.next_line().await? {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
match serde_json::from_str::<Command>(line) {
|
||||
Ok(cmd) => {
|
||||
if cmds.send(cmd).await.is_err() {
|
||||
break; // Worker is gone; so are we.
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(error = %e, line, "bad command"),
|
||||
}
|
||||
}
|
||||
writer.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sends one command to a running daemon and prints the events it produces.
|
||||
pub async fn proxy(path: &Path, cmd: &Command) -> Result<()> {
|
||||
let stream = UnixStream::connect(path).await?;
|
||||
let (read, mut write) = stream.into_split();
|
||||
let mut line = serde_json::to_string(cmd)?;
|
||||
line.push('\n');
|
||||
write.write_all(line.as_bytes()).await?;
|
||||
|
||||
// ponytail: the event stream is a broadcast, so a busy daemon's other work shows up
|
||||
// here too. Fine for a CLI; a UI that cares would want per-request ids.
|
||||
let mut lines = BufReader::new(read).lines();
|
||||
while let Some(line) = lines.next_line().await? {
|
||||
let Ok(ev) = serde_json::from_str::<Event>(&line) else {
|
||||
continue;
|
||||
};
|
||||
if let Some(text) = ev.human() {
|
||||
println!("{text}");
|
||||
}
|
||||
if ev.is_terminal() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn commands_parse_from_the_wire_form() {
|
||||
let got: Command = serde_json::from_str(r#"{"cmd":"fetch"}"#).unwrap();
|
||||
assert!(matches!(got, Command::Fetch { feed: None, force: false }));
|
||||
|
||||
let got: Command = serde_json::from_str(r#"{"cmd":"fetch","feed":"atp","force":true}"#).unwrap();
|
||||
assert!(matches!(got, Command::Fetch { feed: Some(f), force: true } if f == "atp"));
|
||||
|
||||
let got: Command = serde_json::from_str(r#"{"cmd":"reap","dry_run":true}"#).unwrap();
|
||||
assert!(matches!(got, Command::Reap { dry_run: true }));
|
||||
|
||||
assert!(serde_json::from_str::<Command>(r#"{"cmd":"nope"}"#).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn events_serialise_to_the_documented_shape() {
|
||||
let ev = Event::Progress {
|
||||
feed: "atp".into(),
|
||||
url: "https://x/ep.mp3".into(),
|
||||
file: "ep.mp3".into(),
|
||||
done: 10_485_760,
|
||||
total: Some(52_428_800),
|
||||
};
|
||||
let json = serde_json::to_string(&ev).unwrap();
|
||||
assert!(json.starts_with(r#"{"ev":"progress""#), "got {json}");
|
||||
assert!(json.contains(r#""done":10485760"#));
|
||||
|
||||
// total is omitted rather than null when the server sent no length.
|
||||
let ev = Event::Progress {
|
||||
feed: "a".into(),
|
||||
url: "u".into(),
|
||||
file: "f".into(),
|
||||
done: 1,
|
||||
total: None,
|
||||
};
|
||||
assert!(!serde_json::to_string(&ev).unwrap().contains("total"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_completion_events_end_a_client_session() {
|
||||
assert!(Event::ScanDone { feeds: 1 }.is_terminal());
|
||||
assert!(Event::ReapDone { files: 0, bytes: 0 }.is_terminal());
|
||||
assert!(!Event::FeedDone {
|
||||
feed: "a".into(),
|
||||
new: 0,
|
||||
downloaded: 0,
|
||||
failed: 0,
|
||||
torrents: 0
|
||||
}
|
||||
.is_terminal());
|
||||
}
|
||||
}
|
||||
363
src/main.rs
363
src/main.rs
@@ -2,11 +2,14 @@ mod config;
|
||||
mod db;
|
||||
mod download;
|
||||
mod feed;
|
||||
mod ipc;
|
||||
mod retention;
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::{Parser, Subcommand};
|
||||
use ipc::{Command as Cmd, Emitter, Event};
|
||||
use std::path::PathBuf;
|
||||
use tokio::sync::{broadcast, mpsc};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "ipx", version, about = "A headless podcatcher")]
|
||||
@@ -15,6 +18,10 @@ struct Cli {
|
||||
#[arg(long, global = true)]
|
||||
config: Option<PathBuf>,
|
||||
|
||||
/// Do the work here even if a daemon is running
|
||||
#[arg(long, global = true)]
|
||||
local: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
@@ -23,12 +30,6 @@ struct Cli {
|
||||
enum Command {
|
||||
/// Show configured feeds and their state
|
||||
List,
|
||||
/// Delete old or over-quota downloads
|
||||
Reap {
|
||||
/// Show what would go, delete nothing
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
},
|
||||
/// Scan feeds for new entries
|
||||
Fetch {
|
||||
/// Only this feed id
|
||||
@@ -37,36 +38,149 @@ enum Command {
|
||||
#[arg(long)]
|
||||
force: bool,
|
||||
},
|
||||
/// Delete old or over-quota downloads
|
||||
Reap {
|
||||
/// Show what would go, delete nothing
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
},
|
||||
/// Counts of feeds, pending and downloaded enclosures
|
||||
Status,
|
||||
/// Run the scheduler and serve the control socket
|
||||
Daemon,
|
||||
}
|
||||
|
||||
/// Everything a command needs. One per process.
|
||||
struct Ctx {
|
||||
cfg: config::Config,
|
||||
db: db::Db,
|
||||
client: reqwest::Client,
|
||||
out: Emitter,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_env("IPX_LOG")
|
||||
.unwrap_or_else(|_| "ipx=info".into()),
|
||||
)
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
|
||||
let config_path = cli.config.clone().unwrap_or_else(config::config_path);
|
||||
let cfg = config::Config::load(&config_path)?;
|
||||
let db = db::Db::open(&config::data_dir().join("state.db"))?;
|
||||
|
||||
match cli.command {
|
||||
Command::List => list(&cfg, &db, &config_path),
|
||||
// A daemon owns the state; don't have two processes downloading the same thing.
|
||||
let wire_cmd = match &cli.command {
|
||||
Command::Fetch { feed, force } => {
|
||||
// Make room before pulling more down, as the original did per download.
|
||||
report_reap(&retention::run(&cfg, &db, false)?, false);
|
||||
fetch(&cfg, &db, feed.as_deref(), force).await
|
||||
Some(Cmd::Fetch { feed: feed.clone(), force: *force })
|
||||
}
|
||||
Command::Reap { dry_run } => {
|
||||
report_reap(&retention::run(&cfg, &db, dry_run)?, true);
|
||||
Command::Reap { dry_run } => Some(Cmd::Reap { dry_run: *dry_run }),
|
||||
Command::Status => Some(Cmd::Status),
|
||||
Command::List | Command::Daemon => None,
|
||||
};
|
||||
if let Some(cmd) = &wire_cmd
|
||||
&& !cli.local
|
||||
&& ipc::daemon_is_live(&cfg.general.socket).await
|
||||
{
|
||||
return ipc::proxy(&cfg.general.socket, cmd).await;
|
||||
}
|
||||
|
||||
let ctx = Ctx {
|
||||
cfg,
|
||||
db,
|
||||
client: reqwest::Client::builder()
|
||||
.user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
|
||||
.build()?,
|
||||
out: Emitter::terminal(),
|
||||
};
|
||||
|
||||
match cli.command {
|
||||
Command::List => list(&ctx, &config_path),
|
||||
Command::Daemon => daemon(ctx).await,
|
||||
_ => run(&ctx, wire_cmd.expect("only List and Daemon have no wire form")).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(ctx: &Ctx, cmd: Cmd) -> Result<()> {
|
||||
match cmd {
|
||||
Cmd::Fetch { feed, force } => {
|
||||
// Make room before pulling more down, as the original did per download.
|
||||
reap(ctx, false, false)?;
|
||||
fetch(ctx, feed.as_deref(), force).await
|
||||
}
|
||||
Cmd::Reap { dry_run } => reap(ctx, dry_run, true),
|
||||
Cmd::Status => {
|
||||
let (pending, downloaded) = ctx.db.counts()?;
|
||||
ctx.out.emit(Event::Status { feeds: ctx.cfg.feeds.len(), pending, downloaded });
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn list(cfg: &config::Config, db: &db::Db, config_path: &std::path::Path) -> Result<()> {
|
||||
if cfg.feeds.is_empty() {
|
||||
async fn daemon(ctx: Ctx) -> Result<()> {
|
||||
let socket = ctx.cfg.general.socket.clone();
|
||||
if ipc::daemon_is_live(&socket).await {
|
||||
anyhow::bail!("a daemon is already listening on {}", socket.display());
|
||||
}
|
||||
|
||||
let (events, _) = broadcast::channel(1024);
|
||||
let (tx_cmd, mut rx_cmd) = mpsc::channel::<Cmd>(64);
|
||||
let ctx = Ctx { out: Emitter::socket(events.clone(), false), ..ctx };
|
||||
|
||||
let server = tokio::spawn(ipc::serve(socket.clone(), events, tx_cmd));
|
||||
|
||||
// One command at a time: the queue is what keeps two scans from overlapping.
|
||||
let mut ticker = tokio::time::interval(std::time::Duration::from_secs(60));
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
tracing::info!(feeds = ctx.cfg.feeds.len(), "daemon started");
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(cmd) = rx_cmd.recv() => {
|
||||
tracing::info!(?cmd, "command from a client");
|
||||
if let Err(e) = run(&ctx, cmd).await {
|
||||
ctx.out.emit(Event::Error { msg: format!("{e:#}") });
|
||||
}
|
||||
}
|
||||
_ = ticker.tick() => {
|
||||
// Per-feed TTL decides what actually gets polled.
|
||||
if let Err(e) = run(&ctx, Cmd::Fetch { feed: None, force: false }).await {
|
||||
ctx.out.emit(Event::Error { msg: format!("{e:#}") });
|
||||
}
|
||||
}
|
||||
_ = shutdown() => break,
|
||||
}
|
||||
}
|
||||
|
||||
server.abort();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
tracing::info!("daemon stopped");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn shutdown() {
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
let mut term = match signal(SignalKind::terminate()) {
|
||||
Ok(s) => s,
|
||||
Err(_) => return std::future::pending().await,
|
||||
};
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => {}
|
||||
_ = term.recv() => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> {
|
||||
if ctx.cfg.feeds.is_empty() {
|
||||
println!("No feeds configured in {}", config_path.display());
|
||||
return Ok(());
|
||||
}
|
||||
for (id, feed) in &cfg.feeds {
|
||||
let s = db.feed_summary(id)?;
|
||||
for (id, feed) in &ctx.cfg.feeds {
|
||||
let s = ctx.db.feed_summary(id)?;
|
||||
println!("{id} {}", s.title.as_deref().unwrap_or("-"));
|
||||
println!(" url {}", feed.url);
|
||||
println!(" last checked {}", ago(s.last_checked));
|
||||
@@ -78,63 +192,98 @@ fn list(cfg: &config::Config, db: &db::Db, config_path: &std::path::Path) -> Res
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch(
|
||||
cfg: &config::Config,
|
||||
db: &db::Db,
|
||||
only: Option<&str>,
|
||||
force: bool,
|
||||
) -> Result<()> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent(concat!("ipx/", env!("CARGO_PKG_VERSION")))
|
||||
.build()?;
|
||||
|
||||
if let Some(id) = only
|
||||
&& !cfg.feeds.contains_key(id)
|
||||
{
|
||||
anyhow::bail!("no feed with id {id:?}");
|
||||
/// `standalone` false means this is the sweep that runs before a scan: it reports what it
|
||||
/// deleted, but must not emit the terminal ReapDone, or a client waiting on its `fetch`
|
||||
/// would stop reading before the scan had even started.
|
||||
fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> {
|
||||
let r = retention::run(&ctx.cfg, &ctx.db, dry_run)?;
|
||||
for c in r.aged_out.iter().chain(r.over_quota.iter()) {
|
||||
ctx.out.emit(Event::Reaped {
|
||||
path: c.path.clone(),
|
||||
bytes: c.bytes.max(0) as u64,
|
||||
});
|
||||
}
|
||||
|
||||
for (id, feed_cfg) in cfg.feeds.iter().filter(|(id, _)| only.is_none_or(|o| o == *id)) {
|
||||
let state = db.http_state(id)?;
|
||||
|
||||
// TTL: the feed's own <ttl> wins when it is longer than our poll interval.
|
||||
if !force && let Some(last) = state.last_checked {
|
||||
let wait = state.ttl_mins.unwrap_or(0).max(cfg.general.interval_mins) * 60;
|
||||
let due = last + wait as i64;
|
||||
if due > db::now() {
|
||||
println!("{id}: not due for {}", duration((due - db::now()) as u64));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
match scan_one(&client, cfg, db, id, feed_cfg, &state).await {
|
||||
Ok(Some(s)) => println!(
|
||||
"{id}: {} new entries, {} downloaded, {} failed, {} torrents deferred",
|
||||
s.new_entries, s.downloaded, s.failed, s.torrents
|
||||
),
|
||||
Ok(None) => println!("{id}: not modified"),
|
||||
Err(e) => {
|
||||
// One bad feed must not end the scan.
|
||||
let msg = format!("{e:#}");
|
||||
println!("{id}: error: {msg}");
|
||||
db.set_feed_error(id, &feed_cfg.url, &msg)?;
|
||||
}
|
||||
}
|
||||
if standalone {
|
||||
ctx.out.emit(Event::ReapDone {
|
||||
files: r.aged_out.len() + r.over_quota.len(),
|
||||
bytes: r.bytes_freed,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch(ctx: &Ctx, only: Option<&str>, force: bool) -> Result<()> {
|
||||
if let Some(id) = only
|
||||
&& !ctx.cfg.feeds.contains_key(id)
|
||||
{
|
||||
anyhow::bail!("no feed with id {id:?}");
|
||||
}
|
||||
|
||||
let mut scanned = 0;
|
||||
for (id, feed_cfg) in ctx
|
||||
.cfg
|
||||
.feeds
|
||||
.iter()
|
||||
.filter(|(id, _)| only.is_none_or(|o| o == *id))
|
||||
{
|
||||
let state = ctx.db.http_state(id)?;
|
||||
|
||||
// TTL: the feed's own <ttl> wins when it is longer than our poll interval.
|
||||
if !force && let Some(last) = state.last_checked {
|
||||
let wait = state.ttl_mins.unwrap_or(0).max(ctx.cfg.general.interval_mins) * 60;
|
||||
let due = last + wait as i64;
|
||||
if due > db::now() {
|
||||
ctx.out.emit(Event::FeedSkip {
|
||||
feed: id.clone(),
|
||||
reason: format!("not due for {}", duration((due - db::now()) as u64)),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
scanned += 1;
|
||||
ctx.out.emit(Event::FeedStart { feed: id.clone() });
|
||||
match scan_one(ctx, id, feed_cfg, &state).await {
|
||||
Ok(Some(s)) => ctx.out.emit(Event::FeedDone {
|
||||
feed: id.clone(),
|
||||
new: s.new_entries,
|
||||
downloaded: s.downloaded,
|
||||
failed: s.failed,
|
||||
torrents: s.torrents,
|
||||
}),
|
||||
Ok(None) => ctx.out.emit(Event::FeedSkip {
|
||||
feed: id.clone(),
|
||||
reason: "not modified".into(),
|
||||
}),
|
||||
Err(e) => {
|
||||
// One bad feed must not end the scan.
|
||||
let msg = format!("{e:#}");
|
||||
ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() });
|
||||
ctx.db.set_feed_error(id, &feed_cfg.url, &msg)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx.out.emit(Event::ScanDone { feeds: scanned });
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Scan {
|
||||
new_entries: usize,
|
||||
downloaded: usize,
|
||||
failed: usize,
|
||||
torrents: usize,
|
||||
}
|
||||
|
||||
/// Ok(None) means 304.
|
||||
async fn scan_one(
|
||||
client: &reqwest::Client,
|
||||
cfg: &config::Config,
|
||||
db: &db::Db,
|
||||
ctx: &Ctx,
|
||||
id: &str,
|
||||
feed_cfg: &config::Feed,
|
||||
state: &db::HttpState,
|
||||
) -> Result<Option<Scan>> {
|
||||
let fetched = feed::fetch(
|
||||
client,
|
||||
&ctx.client,
|
||||
feed_cfg,
|
||||
state.etag.as_deref(),
|
||||
state.last_modified.as_deref(),
|
||||
@@ -143,14 +292,14 @@ async fn scan_one(
|
||||
|
||||
let (bytes, etag, last_modified) = match fetched {
|
||||
feed::Fetched::NotModified => {
|
||||
db.touch_feed(id, &feed_cfg.url)?;
|
||||
ctx.db.touch_feed(id, &feed_cfg.url)?;
|
||||
return Ok(None);
|
||||
}
|
||||
feed::Fetched::Body { bytes, etag, last_modified } => (bytes, etag, last_modified),
|
||||
};
|
||||
|
||||
let parsed = feed::parse(&bytes)?;
|
||||
db.record_feed(
|
||||
ctx.db.record_feed(
|
||||
id,
|
||||
&feed_cfg.url,
|
||||
parsed.title.as_deref(),
|
||||
@@ -161,42 +310,55 @@ async fn scan_one(
|
||||
|
||||
let mut scan = Scan::default();
|
||||
for entry in &parsed.entries {
|
||||
if db.record_entry(id, entry)? {
|
||||
if ctx.db.record_entry(id, entry)? {
|
||||
scan.new_entries += 1;
|
||||
}
|
||||
for enc in &entry.enclosures {
|
||||
if !db.record_enclosure(id, &entry.guid, enc)? {
|
||||
if !ctx.db.record_enclosure(id, &entry.guid, enc)? {
|
||||
continue; // Seen before: downloaded, skipped or deliberately reaped.
|
||||
}
|
||||
// Filters run once, at discovery, and are recorded in `state`. The download
|
||||
// queue below is then just "everything still pending".
|
||||
if let Some(reason) = reject(feed_cfg, entry, &enc.url) {
|
||||
db.mark_enclosure(&enc.url, "skipped", Some(reason))?;
|
||||
ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let budget = feed_cfg.max_new_per_check.unwrap_or(usize::MAX);
|
||||
if feed_cfg.auto_download && budget > 0 {
|
||||
let folder = download::folder_for(cfg, id, feed_cfg, parsed.title.as_deref());
|
||||
let dest_dir = cfg.general.download_dir.join(&folder);
|
||||
let folder = download::folder_for(&ctx.cfg, id, feed_cfg, parsed.title.as_deref());
|
||||
let dest_dir = ctx.cfg.general.download_dir.join(&folder);
|
||||
|
||||
for item in db.pending(id, budget)? {
|
||||
for item in ctx.db.pending(id, budget)? {
|
||||
if download::looks_like_torrent(&item.url, item.mime.as_deref()) {
|
||||
// Step 7 owns these; recorded so nothing re-queues them meanwhile.
|
||||
db.mark_enclosure(&item.url, "torrent", None)?;
|
||||
ctx.db.mark_enclosure(&item.url, "torrent", None)?;
|
||||
ctx.out.emit(Event::TorrentDeferred {
|
||||
feed: id.to_string(),
|
||||
url: item.url.clone(),
|
||||
});
|
||||
scan.torrents += 1;
|
||||
continue;
|
||||
}
|
||||
match fetch_one(client, cfg, db, feed_cfg, &item.url, &dest_dir).await {
|
||||
Ok(path) => {
|
||||
println!(" saved {}", path.display());
|
||||
match fetch_one(ctx, id, feed_cfg, &item.url, &dest_dir).await {
|
||||
Ok((path, bytes)) => {
|
||||
ctx.out.emit(Event::DownloadDone {
|
||||
feed: id.to_string(),
|
||||
url: item.url.clone(),
|
||||
path: path.display().to_string(),
|
||||
bytes,
|
||||
});
|
||||
scan.downloaded += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("{e:#}");
|
||||
println!(" failed {}: {msg}", item.url);
|
||||
db.mark_enclosure(&item.url, "error", Some(&msg))?;
|
||||
ctx.out.emit(Event::DownloadError {
|
||||
feed: id.to_string(),
|
||||
url: item.url.clone(),
|
||||
msg: msg.clone(),
|
||||
});
|
||||
ctx.db.mark_enclosure(&item.url, "error", Some(&msg))?;
|
||||
scan.failed += 1;
|
||||
}
|
||||
}
|
||||
@@ -206,14 +368,6 @@ async fn scan_one(
|
||||
Ok(Some(scan))
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Scan {
|
||||
new_entries: usize,
|
||||
downloaded: usize,
|
||||
failed: usize,
|
||||
torrents: usize,
|
||||
}
|
||||
|
||||
/// Why this enclosure should not be downloaded, if it should not be.
|
||||
fn reject(feed_cfg: &config::Feed, entry: &feed::Entry, url: &str) -> Option<&'static str> {
|
||||
if !feed_cfg.auto_download {
|
||||
@@ -236,22 +390,27 @@ fn reject(feed_cfg: &config::Feed, entry: &feed::Entry, url: &str) -> Option<&'s
|
||||
}
|
||||
|
||||
async fn fetch_one(
|
||||
client: &reqwest::Client,
|
||||
cfg: &config::Config,
|
||||
db: &db::Db,
|
||||
ctx: &Ctx,
|
||||
feed_id: &str,
|
||||
feed_cfg: &config::Feed,
|
||||
url: &str,
|
||||
dest_dir: &std::path::Path,
|
||||
) -> Result<std::path::PathBuf> {
|
||||
) -> Result<(PathBuf, u64)> {
|
||||
// Throttled to whole percents, as the original's lastDLStepSize guard did.
|
||||
let mut last_pct = -1i64;
|
||||
let name = download::filename_for(url, None);
|
||||
let got = download::download(client, cfg, feed_cfg, url, |done, total| {
|
||||
let got = download::download(&ctx.client, &ctx.cfg, feed_cfg, url, |done, total| {
|
||||
if let Some(t) = total.filter(|t| *t > 0) {
|
||||
let pct = (done * 100 / t) as i64;
|
||||
if pct > last_pct {
|
||||
last_pct = pct;
|
||||
println!("{}", download::progress_line(&name, done, total));
|
||||
ctx.out.emit(Event::Progress {
|
||||
feed: feed_id.to_string(),
|
||||
url: url.to_string(),
|
||||
file: name.clone(),
|
||||
done,
|
||||
total,
|
||||
});
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -260,31 +419,13 @@ async fn fetch_one(
|
||||
if matches!(got.kind, download::Kind::Torrent) {
|
||||
// The MIME lied. Don't file a .torrent as an episode.
|
||||
let _ = tokio::fs::remove_file(&got.tmp).await;
|
||||
db.mark_enclosure(url, "torrent", None)?;
|
||||
ctx.db.mark_enclosure(url, "torrent", None)?;
|
||||
anyhow::bail!("body is a torrent, deferred to the torrent downloader");
|
||||
}
|
||||
|
||||
let path = download::place(&got, dest_dir).await?;
|
||||
db.mark_downloaded(url, &path, got.bytes)?;
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn report_reap(r: &retention::Report, verbose: bool) {
|
||||
for c in r.aged_out.iter().chain(r.over_quota.iter()) {
|
||||
println!("reap {} ({:.1} MB)", c.path, c.bytes.max(0) as f64 / 1_048_576.0);
|
||||
}
|
||||
if r.reconciled > 0 {
|
||||
println!("{} row(s) pointed at files that were already gone", r.reconciled);
|
||||
}
|
||||
let total = r.aged_out.len() + r.over_quota.len();
|
||||
if total > 0 || verbose {
|
||||
println!(
|
||||
"reaped {total} file(s), {:.1} MB, {} stale entr{} pruned",
|
||||
r.bytes_freed as f64 / 1_048_576.0,
|
||||
r.entries_pruned,
|
||||
if r.entries_pruned == 1 { "y" } else { "ies" }
|
||||
);
|
||||
}
|
||||
ctx.db.mark_downloaded(url, &path, got.bytes)?;
|
||||
Ok((path, got.bytes))
|
||||
}
|
||||
|
||||
fn ago(t: Option<i64>) -> String {
|
||||
|
||||
Reference in New Issue
Block a user