DeadAir on the server: copies with long silences shortened (#171)
For anyone who turns DeadAir on, a background task makes each episode a
copy with every silence quieter than -38 dB for over 0.35 s shortened to
0.2 s. Nothing is re-encoded: ffmpeg copies the original's frames and
leaves the silent ones out, about 11 s for an hour of MP3 in a release
build. On Brad & Will's episode 359 that is 520 cuts and 3.3 minutes
saved, the copy decodes clean, and it has no isolated spikes in its
silences that the original lacks.
Cuts are made from the middle of each silence, keeping 0.1 s at each
end, so the frames either side of a cut are silent: decoding blends a
frame with the one before it, and an MP3 frame can begin in the bytes of
the frame before, so the first frame after a cut decodes against the
wrong neighbour. Inside a silence nobody hears that; at a word they would.
Positions stay in the original's time and the server converts them both
ways. The iOS and Android apps save positions themselves from their own
player, even while the page is asleep, so converting in the page would
have stored the copy's times as the original's. /media/{id}/deadair
serves the copy at its own address and records that this person is
playing it; a position saved meanwhile is converted from the copy's
clock, as is the length a player measures.
The copy is made by a task of its own, not the worker: a first pass over
a library is hours of ffmpeg, and scans must not wait behind it. It
checks every two minutes, which covers new downloads, files already
downloaded, a switch turned on and a file downloaded again with one rule.
Episodes everyone who would hear the copy has finished get none.
download::remove deletes a copy with its original, and the quota counts
copies.
Settings: the switch is PATCH /api/me {deadair}, a feed kept out is
PATCH /api/feeds/{id} {deadair: false}. GET /api/me gives the switch and
deadair_saved, the seconds saved on episodes heard 75% or more on a copy,
each counted once. Three new tables, so no ALTER.
The image gains Debian's ffmpeg, about 480 MB, in a layer that stays
cached until the base image changes. The player's switch, the feed
setting and the stat come next, after mockups.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
244
src/db.rs
244
src/db.rs
@@ -2,7 +2,7 @@
|
||||
//! per-feed .ipxd plists, history.dat and qmcache.dat.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use crate::entity::{api_tokens, blocklists, catalogue, enclosures, entries, feeds, hidden, sessions, settings, subscriptions, users};
|
||||
use crate::entity::{api_tokens, blocklists, catalogue, deadair_prefs, enclosures, entries, feeds, hidden, sessions, settings, subscriptions, users};
|
||||
use sea_orm::sea_query::{Expr, Func};
|
||||
use sea_orm::{
|
||||
ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, Set,
|
||||
@@ -60,6 +60,9 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> {
|
||||
schema.create_table_from_entity(blocklists::Entity),
|
||||
schema.create_table_from_entity(hidden::Entity),
|
||||
schema.create_table_from_entity(api_tokens::Entity),
|
||||
schema.create_table_from_entity(deadair::Entity),
|
||||
schema.create_table_from_entity(deadair_prefs::Entity),
|
||||
schema.create_table_from_entity(deadair_saved::Entity),
|
||||
] {
|
||||
orm.execute(table.if_not_exists()).await.context("creating the schema")?;
|
||||
}
|
||||
@@ -198,6 +201,7 @@ impl From<enclosures::Model> for EncRow {
|
||||
fn from(e: enclosures::Model) -> Self {
|
||||
EncRow {
|
||||
id: e.id,
|
||||
deadair: None,
|
||||
feed_id: e.feed_id,
|
||||
guid: e.guid,
|
||||
url: e.url,
|
||||
@@ -299,6 +303,13 @@ impl Db {
|
||||
/// A SQLite file of its own (two connections to one ":memory:" would be two databases); or,
|
||||
/// with IPX_TEST_DATABASE_URL set, a Postgres schema of its own on that database, so the
|
||||
/// same tests prove the SQL on both.
|
||||
/// Runs a statement as it is, for a test outside this module setting up what it needs.
|
||||
#[cfg(test)]
|
||||
pub async fn sql(&self, sql: &str) -> Result<()> {
|
||||
self.orm.execute_unprepared(sql).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn memory() -> Result<Self> {
|
||||
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
|
||||
@@ -1159,6 +1170,10 @@ impl Filter {
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct EncRow {
|
||||
pub id: i64,
|
||||
/// Seconds DeadAir's copy of this file saves, when the person asking plays it from one
|
||||
/// (`deadair::for_reader`). None from the database.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub deadair: Option<i64>,
|
||||
pub feed_id: String,
|
||||
pub guid: String,
|
||||
pub url: String,
|
||||
@@ -2192,6 +2207,188 @@ pub fn step(of: &'static str) -> tracing::Span {
|
||||
if tracing::Span::current().is_none() { tracing::Span::none() } else { tracing::info_span!("step", of) }
|
||||
}
|
||||
|
||||
/// A file on disk with no DeadAir copy made since it was downloaded (issue #171).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DeadAirTodo {
|
||||
pub id: i64,
|
||||
pub feed_id: String,
|
||||
pub guid: String,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
/// An enclosure's DeadAir copy: where it is, what it left out, and how much that came to.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Copy {
|
||||
pub enclosure_id: i64,
|
||||
pub path: String,
|
||||
pub cuts: Vec<crate::deadair::Cut>,
|
||||
pub saved_ms: i64,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// DeadAir settings as (person, feed, wanted), everyone's or one person's. An empty feed is
|
||||
/// the switch beside the speed.
|
||||
pub async fn deadair_prefs(&self, user_id: Option<i64>) -> Result<Vec<(i64, String, bool)>> {
|
||||
let mut a = Args::default();
|
||||
let only = match user_id {
|
||||
Some(u) => format!("WHERE user_id = {}", a.p(u)),
|
||||
None => String::new(),
|
||||
};
|
||||
self.rows(&format!("SELECT user_id, feed_id, wanted FROM deadair_prefs {only}"), a.0)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| Ok((r.try_get("", "user_id")?, r.try_get("", "feed_id")?, r.try_get("", "wanted")?)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Sets one DeadAir setting, or with None takes it away: for a feed, that is following the
|
||||
/// switch (or the feed's group) again.
|
||||
pub async fn set_deadair(&self, user_id: i64, feed_id: &str, wanted: Option<bool>) -> Result<()> {
|
||||
match wanted {
|
||||
None => {
|
||||
deadair_prefs::Entity::delete_by_id((user_id, feed_id.to_owned())).exec(&self.orm).await?;
|
||||
}
|
||||
Some(w) => {
|
||||
self.exec(
|
||||
"INSERT INTO deadair_prefs (user_id, feed_id, wanted) VALUES ($1, $2, $3)
|
||||
ON CONFLICT (user_id, feed_id) DO UPDATE SET wanted = excluded.wanted",
|
||||
vec![user_id.into(), feed_id.into(), w.into()],
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Files on disk with no copy made since they were downloaded, oldest first.
|
||||
pub async fn deadair_todo(&self) -> Result<Vec<DeadAirTodo>> {
|
||||
self.rows(
|
||||
"SELECT e.id, e.feed_id, e.guid, e.path FROM enclosures e
|
||||
LEFT JOIN deadair d ON d.enclosure_id = e.id
|
||||
WHERE e.path IS NOT NULL AND e.state = 'done'
|
||||
AND (d.enclosure_id IS NULL OR d.made_at < coalesce(e.downloaded_at, 0))
|
||||
ORDER BY e.id",
|
||||
vec![],
|
||||
)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| {
|
||||
Ok(DeadAirTodo {
|
||||
id: r.try_get("", "id")?,
|
||||
feed_id: r.try_get("", "feed_id")?,
|
||||
guid: r.try_get("", "guid")?,
|
||||
path: r.try_get("", "path")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Whether any of these people subscribes to the feed.
|
||||
pub async fn subscribed_any(&self, users: &[i64], feed_id: &str) -> Result<bool> {
|
||||
if users.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
let mut a = Args::default();
|
||||
let feed = a.p(feed_id);
|
||||
let ids: Vec<String> = users.iter().map(|&u| a.p(u)).collect();
|
||||
let sql = format!(
|
||||
"SELECT 1 AS one FROM subscriptions WHERE feed_id = {feed} AND user_id IN ({}) LIMIT 1",
|
||||
ids.join(", ")
|
||||
);
|
||||
Ok(!self.rows(&sql, a.0).await?.is_empty())
|
||||
}
|
||||
|
||||
/// Who has an item marked read, which playing it to the end or past 90% does.
|
||||
pub async fn readers(&self, feed_id: &str, guid: &str) -> Result<Vec<i64>> {
|
||||
self.rows(
|
||||
"SELECT user_id FROM entry_state WHERE feed_id = $1 AND guid = $2 AND read",
|
||||
vec![feed_id.into(), guid.into()],
|
||||
)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| Ok(r.try_get("", "user_id")?))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Records the copy made of an enclosure, or with no path that it has none.
|
||||
pub async fn store_deadair(&self, enclosure_id: i64, path: Option<&str>, cuts: &str, saved_ms: i64) -> Result<()> {
|
||||
self.exec(
|
||||
"INSERT INTO deadair (enclosure_id, path, cuts, saved_ms, made_at) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (enclosure_id) DO UPDATE SET path = excluded.path, cuts = excluded.cuts,
|
||||
saved_ms = excluded.saved_ms, made_at = excluded.made_at",
|
||||
vec![enclosure_id.into(), path.map(str::to_owned).into(), cuts.into(), saved_ms.into(), now().into()],
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The copies there are of these enclosures, made since each was last downloaded.
|
||||
pub async fn copies(&self, enclosure_ids: &[i64]) -> Result<Vec<Copy>> {
|
||||
if enclosure_ids.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let mut a = Args::default();
|
||||
let ids: Vec<String> = enclosure_ids.iter().map(|&i| a.p(i)).collect();
|
||||
let sql = format!(
|
||||
"SELECT d.enclosure_id, d.path, d.cuts, d.saved_ms FROM deadair d
|
||||
JOIN enclosures e ON e.id = d.enclosure_id
|
||||
WHERE d.enclosure_id IN ({}) AND d.path IS NOT NULL AND e.path IS NOT NULL
|
||||
AND d.made_at >= coalesce(e.downloaded_at, 0)",
|
||||
ids.join(", ")
|
||||
);
|
||||
self.rows(&sql, a.0)
|
||||
.await?
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let cuts: String = r.try_get("", "cuts")?;
|
||||
Ok(Copy {
|
||||
enclosure_id: r.try_get("", "enclosure_id")?,
|
||||
path: r.try_get("", "path")?,
|
||||
cuts: serde_json::from_str(&cuts).context("reading DeadAir's cuts")?,
|
||||
saved_ms: r.try_get("", "saved_ms")?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Counts an episode towards the time DeadAir has saved someone, once however often it is
|
||||
/// played.
|
||||
pub async fn count_deadair(&self, user_id: i64, feed_id: &str, guid: &str, saved_ms: i64) -> Result<()> {
|
||||
self.exec(
|
||||
"INSERT INTO deadair_saved (user_id, feed_id, guid, saved_ms, at) VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (user_id, feed_id, guid) DO NOTHING",
|
||||
vec![user_id.into(), feed_id.into(), guid.into(), saved_ms.into(), now().into()],
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// An episode's length as someone's player measured it, else as its feed gives it.
|
||||
pub async fn entry_duration(&self, user_id: i64, feed_id: &str, guid: &str) -> Result<Option<i64>> {
|
||||
let rows = self
|
||||
.rows(
|
||||
"SELECT coalesce(s.duration, e.duration) AS duration FROM entries e
|
||||
LEFT JOIN entry_state s ON s.user_id = $1 AND s.feed_id = e.feed_id AND s.guid = e.guid
|
||||
WHERE e.feed_id = $2 AND e.guid = $3",
|
||||
vec![user_id.into(), feed_id.into(), guid.into()],
|
||||
)
|
||||
.await?;
|
||||
Ok(rows.first().map(|r| r.try_get::<Option<i64>>("", "duration")).transpose()?.flatten())
|
||||
}
|
||||
|
||||
/// The time DeadAir has saved someone, in ms.
|
||||
pub async fn deadair_total(&self, user_id: i64) -> Result<i64> {
|
||||
let rows = self
|
||||
.rows(
|
||||
"SELECT CAST(coalesce(sum(saved_ms), 0) AS BIGINT) AS total FROM deadair_saved WHERE user_id = $1",
|
||||
vec![user_id.into()],
|
||||
)
|
||||
.await?;
|
||||
// Cast: Postgres sums bigints as NUMERIC, which does not read as an i64.
|
||||
Ok(rows.first().map(|r| r.try_get::<i64>("", "total")).transpose()?.unwrap_or(0))
|
||||
}
|
||||
}
|
||||
|
||||
/// Unix seconds. Everything time-shaped in the DB is stored this way.
|
||||
pub fn now() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
@@ -2204,6 +2401,51 @@ pub fn now() -> i64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn deadair_copies_are_made_once_per_download_and_counted_once_per_episode() {
|
||||
let db = Db::memory().await.unwrap();
|
||||
let u = db.create_user("dee", None, false).await.unwrap();
|
||||
db.set_deadair(u, "", Some(true)).await.unwrap();
|
||||
db.set_deadair(u, "f", Some(false)).await.unwrap();
|
||||
db.set_deadair(u, "f", None).await.unwrap();
|
||||
assert_eq!(db.deadair_prefs(Some(u)).await.unwrap(), vec![(u, String::new(), true)]);
|
||||
for sql in [
|
||||
"INSERT INTO enclosures (id, feed_id, guid, url, path, state, downloaded_at)
|
||||
VALUES (1, 'f', 'g', 'http://x/a.mp3', '/x/a.mp3', 'done', 100)",
|
||||
"INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (2, 'f', 'h', 'http://x/b.mp3', NULL, 'reaped')",
|
||||
"INSERT INTO entries (feed_id, guid, first_seen, duration) VALUES ('f', 'g', 1, 600)",
|
||||
] {
|
||||
db.orm.execute_unprepared(sql).await.unwrap();
|
||||
}
|
||||
let todo = db.deadair_todo().await.unwrap();
|
||||
assert_eq!(todo.iter().map(|t| t.id).collect::<Vec<_>>(), [1], "only a file on disk");
|
||||
|
||||
db.store_deadair(1, Some("/x/a.deadair.mp3"), "[[1000,3000]]", 2000).await.unwrap();
|
||||
assert!(db.deadair_todo().await.unwrap().is_empty());
|
||||
let c = db.copies(&[1, 2]).await.unwrap();
|
||||
assert_eq!((c.len(), c[0].cuts.as_slice(), c[0].saved_ms), (1, &[crate::deadair::Cut(1000, 3000)][..], 2000));
|
||||
// Downloaded again since: the copy is stale until made again.
|
||||
db.orm.execute_unprepared(&format!("UPDATE enclosures SET downloaded_at = {} WHERE id = 1", now() + 60)).await.unwrap();
|
||||
assert_eq!(db.deadair_todo().await.unwrap().len(), 1);
|
||||
assert!(db.copies(&[1]).await.unwrap().is_empty());
|
||||
|
||||
assert!(!db.subscribed_any(&[u], "f").await.unwrap());
|
||||
db.subscribe(u, "f").await.unwrap();
|
||||
assert!(db.subscribed_any(&[u], "f").await.unwrap());
|
||||
assert!(db.readers("f", "g").await.unwrap().is_empty());
|
||||
db.set_entry_flag(u, "f", "g", EntryFlag::Read, true).await.unwrap();
|
||||
assert_eq!(db.readers("f", "g").await.unwrap(), [u]);
|
||||
|
||||
assert_eq!(db.entry_duration(u, "f", "g").await.unwrap(), Some(600));
|
||||
db.set_position(u, "f", "g", 10, Some(590)).await.unwrap();
|
||||
assert_eq!(db.entry_duration(u, "f", "g").await.unwrap(), Some(590), "what the player measured");
|
||||
|
||||
assert_eq!(db.deadair_total(u).await.unwrap(), 0);
|
||||
db.count_deadair(u, "f", "g", 2000).await.unwrap();
|
||||
db.count_deadair(u, "f", "g", 2000).await.unwrap();
|
||||
assert_eq!(db.deadair_total(u).await.unwrap(), 2000, "a replay counts once");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_read_does_not_wait_for_a_write_in_progress() {
|
||||
use sea_orm::TransactionTrait;
|
||||
|
||||
520
src/deadair.rs
Normal file
520
src/deadair.rs
Normal file
@@ -0,0 +1,520 @@
|
||||
//! DeadAir (issue #171): podcasts play from a copy with their long silences shortened, so they
|
||||
//! finish sooner without sounding sped up.
|
||||
//!
|
||||
//! The copy is the original's own frames with the silent ones left out, copied by ffmpeg with
|
||||
//! nothing re-encoded: about 10 s an hour of audio on one core, finding the silences included.
|
||||
//! It is made in the background, one file at a time, for files in feeds someone has DeadAir on
|
||||
//! for, and kept beside the original (`copy_path`) until `download::remove` takes both.
|
||||
//!
|
||||
//! Positions stay in the original's time. What the copy left out is stored as `Cut`s of the
|
||||
//! original's time, and the server converts both ways (`to_copy`, `to_original`): the iOS and
|
||||
//! Android apps save positions themselves, from their own player, so converting in the page
|
||||
//! would miss them.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::Ctx;
|
||||
|
||||
/// Quieter than this for longer than `MIN_SILENCE` is a silence: what the measurements on the
|
||||
/// issue used. A podcast's room tone sits well below it and speech well above.
|
||||
const THRESHOLD: &str = "-38dB";
|
||||
const MIN_SILENCE: f64 = 0.35;
|
||||
/// Kept at each end of a silence, so the frames on both sides of a cut are silent. Decoding
|
||||
/// blends a frame with the one before it, and an MP3 frame can begin in the bytes of the frame
|
||||
/// before (the bit reservoir): the first frame after a cut decodes against the wrong neighbour,
|
||||
/// which nobody hears between two silent frames and everyone hears at the start of a word.
|
||||
const KEEP: f64 = 0.1;
|
||||
/// ffmpeg takes the frames to drop as one argument, and Linux refuses an argument over 128 KiB.
|
||||
// shortcut: a file with more silences than this keeps only its longest; about 9 hours of
|
||||
// conversation at the rate measured. Pass the list some other way if a feed ever gets there.
|
||||
const MAX_CUTS: usize = 4000;
|
||||
/// How long the maker waits when there is nothing to do: a download or a switch turned on is
|
||||
/// picked up this long after, at most.
|
||||
const IDLE: std::time::Duration = std::time::Duration::from_secs(120);
|
||||
|
||||
/// A span of the original, in milliseconds, that the copy leaves out.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Cut(pub i64, pub i64);
|
||||
|
||||
/// Where the copy of `original` goes: beside it, `name.deadair.mp3` for `name.mp3`.
|
||||
pub fn copy_path(original: &Path) -> PathBuf {
|
||||
let stem = original.file_stem().map(|s| s.to_string_lossy()).unwrap_or_default();
|
||||
let name = match original.extension() {
|
||||
Some(ext) => format!("{stem}.deadair.{}", ext.to_string_lossy()),
|
||||
None => format!("{stem}.deadair"),
|
||||
};
|
||||
original.with_file_name(name)
|
||||
}
|
||||
|
||||
/// The silences silencedetect reported, as (start, end) seconds. One still open at the end of
|
||||
/// the file has no end and is left alone.
|
||||
pub fn parse_silences(log: &str) -> Vec<(f64, f64)> {
|
||||
let mut out = Vec::new();
|
||||
let mut start = None;
|
||||
for line in log.lines() {
|
||||
if let Some(v) = after(line, "silence_start: ") {
|
||||
start = Some(v);
|
||||
} else if let Some(v) = after(line, "silence_end: ")
|
||||
&& let Some(s) = start.take()
|
||||
{
|
||||
out.push((s, v));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn after(line: &str, key: &str) -> Option<f64> {
|
||||
let rest = &line[line.find(key)? + key.len()..];
|
||||
rest.split_whitespace().next()?.parse().ok()
|
||||
}
|
||||
|
||||
/// Which frames to drop for these silences, as inclusive ranges of frame numbers, and the same
|
||||
/// as `Cut`s. Frame `n` plays from `first + n * frame` seconds. Only frames wholly inside a
|
||||
/// silence, `KEEP` in from each end, go.
|
||||
pub fn plan(silences: &[(f64, f64)], first: f64, frame: f64) -> (Vec<(u64, u64)>, Vec<Cut>) {
|
||||
let mut spans: Vec<(u64, u64)> = silences
|
||||
.iter()
|
||||
.filter(|(s, e)| e - s >= MIN_SILENCE)
|
||||
.filter_map(|&(s, e)| {
|
||||
let a = ((s + KEEP - first) / frame).ceil().max(0.0) as u64;
|
||||
let b = ((e - KEEP - first) / frame).floor() as i64 - 1;
|
||||
(b >= a as i64).then_some((a, b as u64))
|
||||
})
|
||||
.collect();
|
||||
if spans.len() > MAX_CUTS {
|
||||
spans.sort_by_key(|&(a, b)| std::cmp::Reverse(b - a));
|
||||
spans.truncate(MAX_CUTS);
|
||||
spans.sort();
|
||||
}
|
||||
let ms = |n: u64| ((first + n as f64 * frame) * 1000.0).round() as i64;
|
||||
let cuts = spans.iter().map(|&(a, b)| Cut(ms(a), ms(b + 1))).collect();
|
||||
(spans, cuts)
|
||||
}
|
||||
|
||||
/// A time in the original, in ms, as the same moment in the copy. A time inside a cut is where
|
||||
/// the cut was made.
|
||||
pub fn to_copy(cuts: &[Cut], ms: i64) -> i64 {
|
||||
let mut out = ms;
|
||||
for &Cut(s, e) in cuts {
|
||||
if ms >= e {
|
||||
out -= e - s;
|
||||
} else {
|
||||
if ms > s {
|
||||
out -= ms - s;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A time in the copy, in ms, as the same moment in the original.
|
||||
pub fn to_original(cuts: &[Cut], ms: i64) -> i64 {
|
||||
let mut out = ms;
|
||||
for &Cut(s, e) in cuts {
|
||||
if out >= s {
|
||||
out += e - s;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn saved(cuts: &[Cut]) -> i64 {
|
||||
cuts.iter().map(|Cut(s, e)| e - s).sum()
|
||||
}
|
||||
|
||||
/// Whether someone with these settings plays a feed from its copies: their switch on, and the
|
||||
/// feed not kept out, by itself or by the group it came in (an OPML or a Patreon creator).
|
||||
pub fn wants(prefs: &HashMap<String, bool>, feed: &str, group: Option<&str>) -> bool {
|
||||
prefs.get("").copied().unwrap_or(false)
|
||||
&& prefs.get(feed).or_else(|| group.and_then(|g| prefs.get(g))).copied().unwrap_or(true)
|
||||
}
|
||||
|
||||
/// Each person's settings who has the switch on, and each managed feed's group, which is what
|
||||
/// `wants` needs.
|
||||
pub struct Wanting {
|
||||
pub people: HashMap<i64, HashMap<String, bool>>,
|
||||
pub groups: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl Wanting {
|
||||
pub async fn load(ctx: &Ctx, only: Option<i64>) -> Result<Self> {
|
||||
let mut people: HashMap<i64, HashMap<String, bool>> = HashMap::new();
|
||||
for (user, feed, wanted) in ctx.db.deadair_prefs(only).await? {
|
||||
people.entry(user).or_default().insert(feed, wanted);
|
||||
}
|
||||
people.retain(|_, p| p.get("").copied().unwrap_or(false));
|
||||
let groups = if people.is_empty() {
|
||||
HashMap::new()
|
||||
} else {
|
||||
ctx.db.managed_feeds().await?.into_iter().map(|m| (m.id, m.group_id)).collect()
|
||||
};
|
||||
Ok(Self { people, groups })
|
||||
}
|
||||
|
||||
pub fn wants(&self, user: i64, feed: &str) -> bool {
|
||||
self.people
|
||||
.get(&user)
|
||||
.is_some_and(|p| wants(p, feed, self.groups.get(feed).map(String::as_str)))
|
||||
}
|
||||
}
|
||||
|
||||
/// What each person is playing from a copy, by (person, feed, guid): set when `/media` serves
|
||||
/// them a copy and cleared when it serves the original, so a saved position is converted from
|
||||
/// the clock it was measured in, whether the page saved it or a native app did.
|
||||
// shortcut: in memory, so a restart forgets it. A position saved before the player next asks
|
||||
// for more of the copy is then taken as the original's, which resumes a little early, never late.
|
||||
pub type Playing = Arc<std::sync::Mutex<HashMap<(i64, String, String), Arc<crate::db::Copy>>>>;
|
||||
|
||||
/// Puts a page of entries in the clock of what this person will play: for each file with a
|
||||
/// copy they play, `deadair` is the seconds it saves, and the entry's position and length are
|
||||
/// the copy's. The page plays `/media/{id}/deadair` for a file that has it.
|
||||
pub async fn for_reader(ctx: &Ctx, user: i64, rows: &mut [crate::db::EntryRow]) -> Result<()> {
|
||||
let wanting = Wanting::load(ctx, Some(user)).await?;
|
||||
if wanting.people.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let ids: Vec<i64> = rows
|
||||
.iter()
|
||||
.filter(|r| wanting.wants(user, &r.feed_id))
|
||||
.flat_map(|r| r.enclosures.iter().filter(|e| e.path.is_some()).map(|e| e.id))
|
||||
.collect();
|
||||
let copies: HashMap<i64, crate::db::Copy> =
|
||||
ctx.db.copies(&ids).await?.into_iter().map(|c| (c.enclosure_id, c)).collect();
|
||||
for r in rows.iter_mut() {
|
||||
let mut converted = false;
|
||||
for e in &mut r.enclosures {
|
||||
let Some(c) = copies.get(&e.id) else { continue };
|
||||
e.deadair = Some(c.saved_ms / 1000);
|
||||
// One position an item, however many files: the first with a copy sets the clock.
|
||||
if !std::mem::replace(&mut converted, true) {
|
||||
r.position = to_copy(&c.cuts, r.position * 1000) / 1000;
|
||||
r.duration = r.duration.map(|d| to_copy(&c.cuts, d * 1000) / 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A position and length a player saved, in the clock of what it was playing, as the
|
||||
/// original's. Past 75% of a copy, the episode also counts towards the time DeadAir has saved
|
||||
/// this person.
|
||||
pub async fn on_save(
|
||||
ctx: &Ctx,
|
||||
playing: &Playing,
|
||||
user: i64,
|
||||
feed: &str,
|
||||
guid: &str,
|
||||
secs: i64,
|
||||
duration: Option<i64>,
|
||||
) -> Result<(i64, Option<i64>)> {
|
||||
let copy = playing.lock().unwrap().get(&(user, feed.to_owned(), guid.to_owned())).cloned();
|
||||
let Some(c) = copy else { return Ok((secs, duration)) };
|
||||
let original = |s: i64| to_original(&c.cuts, s * 1000) / 1000;
|
||||
let (secs, duration) = (original(secs), duration.map(original));
|
||||
let len = match duration {
|
||||
Some(d) => Some(d),
|
||||
None => ctx.db.entry_duration(user, feed, guid).await?,
|
||||
};
|
||||
if let Some(len) = len
|
||||
&& len > 0
|
||||
&& secs * 4 >= len * 3
|
||||
{
|
||||
ctx.db.count_deadair(user, feed, guid, c.saved_ms).await?;
|
||||
}
|
||||
Ok((secs, duration))
|
||||
}
|
||||
|
||||
/// Makes copies until stopped: the next one anyone wants, or a wait of `IDLE` when there is
|
||||
/// none.
|
||||
pub async fn run(ctx: Arc<Ctx>, mut stop: tokio::sync::watch::Receiver<bool>) {
|
||||
if !have_ffmpeg().await {
|
||||
tracing::warn!("ffmpeg is not installed, so DeadAir makes no copies");
|
||||
return;
|
||||
}
|
||||
loop {
|
||||
// Dropping the pass on a signal kills the ffmpeg it is running (`kill_on_drop`).
|
||||
let made = tokio::select! {
|
||||
_ = stop.changed() => return,
|
||||
made = next(&ctx) => made,
|
||||
};
|
||||
match made {
|
||||
Ok(true) => continue,
|
||||
Ok(false) => {}
|
||||
Err(e) => tracing::warn!(error = %format!("{e:#}"), "DeadAir could not make a copy"),
|
||||
}
|
||||
tokio::select! {
|
||||
_ = stop.changed() => return,
|
||||
_ = tokio::time::sleep(IDLE) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn have_ffmpeg() -> bool {
|
||||
tokio::process::Command::new("ffmpeg")
|
||||
.arg("-version")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.status()
|
||||
.await
|
||||
.is_ok_and(|s| s.success())
|
||||
}
|
||||
|
||||
/// Makes one copy someone wants, if there is one to make. A file everyone who would play the
|
||||
/// copy has already finished gets none: finished episodes are rarely played again.
|
||||
// shortcut: those stay on the list and are looked at again each pass, one query each; record
|
||||
// them as passed over if a large library makes the pass slow.
|
||||
async fn next(ctx: &Ctx) -> Result<bool> {
|
||||
// Nobody with the switch on, which is most passes: nothing more to ask the database.
|
||||
let wanting = Wanting::load(ctx, None).await?;
|
||||
if wanting.people.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
for t in ctx.db.deadair_todo().await? {
|
||||
let users: Vec<i64> =
|
||||
wanting.people.keys().copied().filter(|&u| wanting.wants(u, &t.feed_id)).collect();
|
||||
if users.is_empty() || !ctx.db.subscribed_any(&users, &t.feed_id).await? {
|
||||
continue;
|
||||
}
|
||||
let readers = ctx.db.readers(&t.feed_id, &t.guid).await?;
|
||||
if users.iter().all(|u| readers.contains(u)) {
|
||||
continue;
|
||||
}
|
||||
let src = crate::download::playable(Path::new(&t.path));
|
||||
let started = std::time::Instant::now();
|
||||
let made = make(&src).await;
|
||||
let (path, cuts) = match made {
|
||||
Ok(Some((path, cuts))) => (Some(path.display().to_string()), cuts),
|
||||
Ok(None) => (None, vec![]),
|
||||
Err(e) => {
|
||||
// Recorded as having none, so a file ffmpeg cannot read is not tried every pass.
|
||||
tracing::warn!(path = %src.display(), error = %format!("{e:#}"), "DeadAir could not make a copy");
|
||||
(None, vec![])
|
||||
}
|
||||
};
|
||||
let saved_ms = saved(&cuts);
|
||||
ctx.db.store_deadair(t.id, path.as_deref(), &serde_json::to_string(&cuts)?, saved_ms).await?;
|
||||
tracing::info!(
|
||||
ev = "deadair",
|
||||
feed = t.feed_id,
|
||||
path = %src.display(),
|
||||
cuts = cuts.len(),
|
||||
saved_s = saved_ms / 1000,
|
||||
took_ms = started.elapsed().as_millis() as u64,
|
||||
"DeadAir copy made"
|
||||
);
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Makes the copy of one file. None for a file it does not apply to: video, anything but MP3
|
||||
/// and AAC, or one with no silence long enough to cut.
|
||||
pub async fn make(src: &Path) -> Result<Option<(PathBuf, Vec<Cut>)>> {
|
||||
let Some((first, frame)) = probe(src).await? else { return Ok(None) };
|
||||
let log = ffmpeg(&[
|
||||
"-hide_banner", "-nostats", "-i", &src.display().to_string(), "-map", "0:a:0",
|
||||
"-af", &format!("silencedetect=n={THRESHOLD}:d={MIN_SILENCE}"), "-f", "null", "-",
|
||||
])
|
||||
.await
|
||||
.context("finding the silences")?;
|
||||
let (frames, cuts) = plan(&parse_silences(&log), first, frame);
|
||||
if cuts.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let drop: Vec<String> = frames.iter().map(|(a, b)| format!("between(n\\,{a}\\,{b})")).collect();
|
||||
let dst = copy_path(src);
|
||||
// Written under another name and moved into place, so a copy cut short by a restart is
|
||||
// never served.
|
||||
let tmp = dst.with_file_name(format!(
|
||||
"{}.part.{}",
|
||||
dst.file_stem().unwrap_or_default().to_string_lossy(),
|
||||
dst.extension().unwrap_or_default().to_string_lossy()
|
||||
));
|
||||
ffmpeg(&[
|
||||
"-v", "error", "-y", "-i", &src.display().to_string(), "-map", "0:a:0", "-c", "copy",
|
||||
"-bsf:a", &format!("noise=drop='{}',setts=ts=N*DURATION", drop.join("+")),
|
||||
&tmp.display().to_string(),
|
||||
])
|
||||
.await
|
||||
.context("copying the frames kept")?;
|
||||
tokio::fs::rename(&tmp, &dst).await.with_context(|| format!("moving into {}", dst.display()))?;
|
||||
Ok(Some((dst, cuts)))
|
||||
}
|
||||
|
||||
/// When the first audio frame starts and how long each lasts, in seconds, for a file DeadAir
|
||||
/// can copy; None for one it cannot. A cover picture in an MP3 is not video.
|
||||
async fn probe(src: &Path) -> Result<Option<(f64, f64)>> {
|
||||
let path = src.display().to_string();
|
||||
let out = run_out("ffprobe", &[
|
||||
"-v", "error", "-show_entries", "stream=codec_type,codec_name:stream_disposition=attached_pic",
|
||||
"-of", "json", &path,
|
||||
])
|
||||
.await?;
|
||||
let v: serde_json::Value = serde_json::from_str(&out).context("reading ffprobe")?;
|
||||
let streams = v["streams"].as_array().cloned().unwrap_or_default();
|
||||
let video = streams
|
||||
.iter()
|
||||
.any(|s| s["codec_type"] == "video" && s["disposition"]["attached_pic"] != 1);
|
||||
let audio = streams.iter().find(|s| s["codec_type"] == "audio");
|
||||
if video || !audio.is_some_and(|a| a["codec_name"] == "mp3" || a["codec_name"] == "aac") {
|
||||
return Ok(None);
|
||||
}
|
||||
let out = run_out("ffprobe", &[
|
||||
"-v", "error", "-select_streams", "a:0", "-show_entries", "packet=pts_time,duration_time",
|
||||
"-read_intervals", "%+#1", "-of", "csv=p=0", &path,
|
||||
])
|
||||
.await?;
|
||||
let mut f = out.trim().split(',').map(|x| x.trim().parse::<f64>());
|
||||
match (f.next(), f.next()) {
|
||||
(Some(Ok(first)), Some(Ok(frame))) if frame > 0.0 => Ok(Some((first, frame))),
|
||||
_ => bail!("no audio frame in {path}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs ffmpeg and gives back what it wrote to stderr, where silencedetect reports.
|
||||
async fn ffmpeg(args: &[&str]) -> Result<String> {
|
||||
let out = command("ffmpeg").args(["-nostdin", "-threads", "1"]).args(args).output().await?;
|
||||
let log = String::from_utf8_lossy(&out.stderr).into_owned();
|
||||
if !out.status.success() {
|
||||
bail!("ffmpeg failed: {}", log.lines().rev().take(3).collect::<Vec<_>>().join(" / "));
|
||||
}
|
||||
Ok(log)
|
||||
}
|
||||
|
||||
async fn run_out(prog: &str, args: &[&str]) -> Result<String> {
|
||||
let out = command(prog).args(args).output().await.with_context(|| format!("running {prog}"))?;
|
||||
if !out.status.success() {
|
||||
bail!("{prog} failed: {}", String::from_utf8_lossy(&out.stderr).trim());
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||
}
|
||||
|
||||
fn command(prog: &str) -> tokio::process::Command {
|
||||
let mut c = tokio::process::Command::new(prog);
|
||||
c.kill_on_drop(true).stdin(std::process::Stdio::null());
|
||||
c
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn silences_are_read_from_the_log_and_an_open_one_is_left() {
|
||||
let log = "[silencedetect @ 0x1] silence_start: 1\n\
|
||||
[silencedetect @ 0x1] silence_end: 3.00002 | silence_duration: 2.00002\n\
|
||||
[silencedetect @ 0x1] silence_start: 10.5\n";
|
||||
assert_eq!(parse_silences(log), vec![(1.0, 3.00002)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cuts_leave_silent_frames_either_side_and_skip_short_silences() {
|
||||
// MP3 frames, 26.122 ms; one silence of 2 s, one too short to touch.
|
||||
let frame = 1152.0 / 44100.0;
|
||||
let (frames, cuts) = plan(&[(1.0, 3.0), (5.0, 5.3)], 0.0, frame);
|
||||
assert_eq!(frames, vec![(43, 110)]);
|
||||
let Cut(s, e) = cuts[0];
|
||||
assert!(s >= 1100 && e <= 2900, "{cuts:?}");
|
||||
assert!(s - 1100 < 27 && 2900 - e < 27, "no more than a frame kept beyond KEEP: {cuts:?}");
|
||||
// AAC in MP4 starts a frame early, for the encoder's priming.
|
||||
let (frames, _) = plan(&[(1.0, 3.0)], -0.02322, 1024.0 / 44100.0);
|
||||
assert_eq!(frames, vec![(49, 124)]);
|
||||
// A silence the margins use up entirely is left alone.
|
||||
assert!(plan(&[(1.0, 1.21)], 0.0, frame).0.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positions_convert_both_ways() {
|
||||
let cuts = [Cut(10_000, 20_000), Cut(30_000, 35_000)];
|
||||
assert_eq!(to_copy(&cuts, 5_000), 5_000);
|
||||
assert_eq!(to_copy(&cuts, 15_000), 10_000, "inside a cut is where it was made");
|
||||
assert_eq!(to_copy(&cuts, 25_000), 15_000);
|
||||
assert_eq!(to_copy(&cuts, 40_000), 25_000);
|
||||
for t in [0, 5_000, 10_000, 15_000, 19_999, 20_000, 25_000, 40_000] {
|
||||
let back = to_original(&cuts, to_copy(&cuts, t));
|
||||
// A time inside a cut comes back as the cut's end: the copy has nothing between.
|
||||
let expect = if (10_000..20_000).contains(&t) { 20_000 } else { t };
|
||||
assert_eq!(back, expect, "{t}");
|
||||
}
|
||||
assert_eq!(saved(&cuts), 15_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_feed_kept_out_by_itself_or_its_group() {
|
||||
let p = |v: &[(&str, bool)]| v.iter().map(|(k, w)| (k.to_string(), *w)).collect::<HashMap<_, _>>();
|
||||
assert!(!wants(&p(&[]), "a", None), "off until turned on");
|
||||
assert!(wants(&p(&[("", true)]), "a", None));
|
||||
assert!(!wants(&p(&[("", true), ("a", false)]), "a", None));
|
||||
assert!(!wants(&p(&[("", true), ("g", false)]), "a", Some("g")));
|
||||
assert!(wants(&p(&[("", true), ("g", false), ("a", true)]), "a", Some("g")), "let back in");
|
||||
assert!(!wants(&p(&[("", false), ("a", true)]), "a", None), "the switch rules");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_copy_sits_beside_the_original() {
|
||||
assert_eq!(copy_path(Path::new("/d/show/ep 1.mp3")), Path::new("/d/show/ep 1.deadair.mp3"));
|
||||
}
|
||||
|
||||
/// The whole of it, on a clip ffmpeg makes: a second of tone, two of silence, and so on.
|
||||
/// Skipped where ffmpeg is not installed.
|
||||
#[tokio::test]
|
||||
async fn a_copy_is_shorter_by_what_was_cut_and_plays() {
|
||||
if !have_ffmpeg().await {
|
||||
eprintln!("no ffmpeg; skipped");
|
||||
return;
|
||||
}
|
||||
let dir = std::env::temp_dir().join(format!("ipx-deadair-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
for (ext, codec) in [("mp3", "libmp3lame"), ("m4a", "aac")] {
|
||||
let src = dir.join(format!("t.{ext}"));
|
||||
ffmpeg(&[
|
||||
"-v", "error", "-y", "-f", "lavfi", "-i",
|
||||
"aevalsrc='if(lt(mod(t,3),1),0.5*sin(2*PI*440*t),0)':d=12:s=44100",
|
||||
"-c:a", codec, &src.display().to_string(),
|
||||
])
|
||||
.await
|
||||
.unwrap();
|
||||
let (dst, cuts) = make(&src).await.unwrap().expect("a copy");
|
||||
assert_eq!(cuts.len(), 4, "{ext}: {cuts:?}");
|
||||
// About 1.8 s from each of the four 2 s silences.
|
||||
let cut = saved(&cuts);
|
||||
assert!((7_000..7_300).contains(&cut), "{ext}: {cut}");
|
||||
let len = |p: &Path| {
|
||||
let p = p.display().to_string();
|
||||
async move {
|
||||
run_out("ffprobe", &["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", &p])
|
||||
.await
|
||||
.unwrap()
|
||||
.trim()
|
||||
.parse::<f64>()
|
||||
.unwrap()
|
||||
}
|
||||
};
|
||||
let short = len(&src).await - len(&dst).await;
|
||||
assert!((short * 1000.0 - cut as f64).abs() < 100.0, "{ext}: {short} s shorter, {cut} ms cut");
|
||||
// Decodes without a complaint.
|
||||
assert!(ffmpeg(&["-v", "error", "-i", &dst.display().to_string(), "-f", "null", "-"]).await.unwrap().is_empty());
|
||||
}
|
||||
std::fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
/// A real episode, for trying the cut on something other than a test tone:
|
||||
/// `IPX_DEADAIR_FILE=/path/ep.mp3 cargo test real_file -- --ignored --nocapture`. The copy is left
|
||||
/// beside it to listen to.
|
||||
#[cfg(test)]
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn real_file() {
|
||||
let src = PathBuf::from(std::env::var("IPX_DEADAIR_FILE").expect("IPX_DEADAIR_FILE"));
|
||||
let t = std::time::Instant::now();
|
||||
let (dst, cuts) = make(&src).await.unwrap().expect("a copy");
|
||||
eprintln!("{} cuts, {:.1} min saved, {:.1?} -> {}", cuts.len(), saved(&cuts) as f64 / 60_000.0, t.elapsed(), dst.display());
|
||||
}
|
||||
@@ -211,7 +211,12 @@ pub async fn place(d: &Downloaded, dir: &Path) -> Result<PathBuf> {
|
||||
/// down as. `remove_file` refused the folder, so the reaper warned on every pass and freed nothing,
|
||||
/// and Delete failed (#158).
|
||||
pub fn remove(path: &Path) -> std::io::Result<()> {
|
||||
if path.is_dir() { std::fs::remove_dir_all(path) } else { std::fs::remove_file(path) }
|
||||
if path.is_dir() {
|
||||
return std::fs::remove_dir_all(path);
|
||||
}
|
||||
// Its DeadAir copy goes with it, if it has one; it is no use without the original (#171).
|
||||
let _ = std::fs::remove_file(crate::deadair::copy_path(path));
|
||||
std::fs::remove_file(path)
|
||||
}
|
||||
|
||||
/// The file to play for an enclosure's path: the path itself, or for a torrent of several files,
|
||||
|
||||
@@ -354,3 +354,70 @@ pub mod settings {
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
/// DeadAir's copy of an enclosure: the same file with long silences shortened (issue #171).
|
||||
/// `cuts` is what the copy leaves out, as `[[start_ms, end_ms], ...]` of the original's time,
|
||||
/// which is what converts positions between the two. `path` is NULL for a file that cannot have
|
||||
/// one (video, or not MP3 or AAC), so it is looked at once and not every pass.
|
||||
pub mod deadair {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "deadair")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub enclosure_id: i64,
|
||||
#[sea_orm(column_type = "Text", nullable)]
|
||||
pub path: Option<String>,
|
||||
#[sea_orm(column_type = "Text")]
|
||||
pub cuts: String,
|
||||
pub saved_ms: i64,
|
||||
/// Against the enclosure's `downloaded_at`: a file downloaded again needs its copy again.
|
||||
pub made_at: i64,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
}
|
||||
|
||||
/// Who has DeadAir on: `feed_id` empty is the switch beside the speed, anything else a feed
|
||||
/// they keep out of it (`wanted` false) or let back in under a group kept out, as blocklists use
|
||||
/// an empty feed for everywhere.
|
||||
pub mod deadair_prefs {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "deadair_prefs")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub user_id: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
|
||||
pub feed_id: String,
|
||||
pub wanted: bool,
|
||||
}
|
||||
|
||||
owned_by_user!();
|
||||
}
|
||||
|
||||
/// The time DeadAir has saved someone: one row per episode they got 75% through on a copy, so
|
||||
/// replaying it never counts twice. Their total is the sum.
|
||||
pub mod deadair_saved {
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "deadair_saved")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key, auto_increment = false)]
|
||||
pub user_id: i64,
|
||||
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
|
||||
pub feed_id: String,
|
||||
#[sea_orm(primary_key, auto_increment = false, column_type = "Text")]
|
||||
pub guid: String,
|
||||
pub saved_ms: i64,
|
||||
pub at: i64,
|
||||
}
|
||||
|
||||
owned_by_user!();
|
||||
}
|
||||
|
||||
58
src/main.rs
58
src/main.rs
@@ -3,6 +3,7 @@ mod art;
|
||||
mod auth;
|
||||
mod config;
|
||||
mod db;
|
||||
mod deadair;
|
||||
mod entity;
|
||||
mod download;
|
||||
mod feed;
|
||||
@@ -635,6 +636,9 @@ async fn daemon(
|
||||
shutdown().await;
|
||||
let _ = tx_stop.send(true);
|
||||
});
|
||||
// Off the worker: the first pass over a library is hours of ffmpeg, and scans and downloads
|
||||
// must not wait behind it.
|
||||
tokio::spawn(deadair::run(ctx.clone(), rx_stop.clone()));
|
||||
|
||||
// Runs one job, abandoning it if a signal arrives. Returns false to end the loop.
|
||||
async fn until_stopped(
|
||||
@@ -765,6 +769,7 @@ async fn start_web(
|
||||
cmds: cmds.clone(),
|
||||
events: events.clone(),
|
||||
access,
|
||||
playing: Default::default(),
|
||||
};
|
||||
Ok(Some(tokio::spawn(async move {
|
||||
if let Err(e) = web::serve(state, &bind).await {
|
||||
@@ -2478,6 +2483,59 @@ mod tests {
|
||||
assert!(ctx.db.subscription(stays, "shared").await.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deadair_hands_out_the_copys_clock_and_saves_in_the_originals() {
|
||||
let ctx = test_ctx(config::Config::default()).await;
|
||||
let db = &ctx.db;
|
||||
let u = db.create_user("dee", None, false).await.unwrap();
|
||||
db.subscribe(u, "f").await.unwrap();
|
||||
for sql in [
|
||||
"INSERT INTO entries (feed_id, guid, first_seen, duration) VALUES ('f', 'g', 1, 600)",
|
||||
"INSERT INTO enclosures (id, feed_id, guid, url, path, state, downloaded_at)
|
||||
VALUES (7, 'f', 'g', 'http://x/a.mp3', '/x/a.mp3', 'done', 100)",
|
||||
] {
|
||||
db.sql(sql).await.unwrap();
|
||||
}
|
||||
// Two minutes cut, both before the place saved.
|
||||
db.store_deadair(7, Some("/x/a.deadair.mp3"), "[[60000,120000],[200000,260000]]", 120_000).await.unwrap();
|
||||
db.set_position(u, "f", "g", 300, None).await.unwrap();
|
||||
let order = db::order_sql("published", "desc");
|
||||
let page = || {
|
||||
let (order, ctx) = (order.clone(), &ctx);
|
||||
async move {
|
||||
let mut rows = ctx.db.entries_in(u, Some("f"), db::Filter::All, None, 0, 10, &order).await.unwrap();
|
||||
deadair::for_reader(ctx, u, &mut rows).await.unwrap();
|
||||
rows.remove(0)
|
||||
}
|
||||
};
|
||||
let off = page().await;
|
||||
assert_eq!((off.position, off.enclosures[0].deadair), (300, None), "off until turned on");
|
||||
|
||||
db.set_deadair(u, "", Some(true)).await.unwrap();
|
||||
let on = page().await;
|
||||
assert_eq!((on.position, on.duration, on.enclosures[0].deadair), (180, Some(480), Some(120)));
|
||||
|
||||
// Saved from the copy at 3:20 of its 8:00: 5:20 of the original's 10:00, short of 75%.
|
||||
let playing = deadair::Playing::default();
|
||||
let copy = db.copies(&[7]).await.unwrap().remove(0);
|
||||
playing.lock().unwrap().insert((u, "f".into(), "g".into()), Arc::new(copy));
|
||||
let saved = deadair::on_save(&ctx, &playing, u, "f", "g", 200, Some(480)).await.unwrap();
|
||||
assert_eq!(saved, (320, Some(600)));
|
||||
assert_eq!(db.deadair_total(u).await.unwrap(), 0);
|
||||
// Past 75% (6:00 of the copy is 8:00 of the original), counted once.
|
||||
for _ in 0..2 {
|
||||
deadair::on_save(&ctx, &playing, u, "f", "g", 360, None).await.unwrap();
|
||||
}
|
||||
assert_eq!(db.deadair_total(u).await.unwrap(), 120_000);
|
||||
// Playing the original again: saved as it is.
|
||||
playing.lock().unwrap().clear();
|
||||
assert_eq!(deadair::on_save(&ctx, &playing, u, "f", "g", 200, Some(600)).await.unwrap(), (200, Some(600)));
|
||||
|
||||
// Kept out: the original's clock again.
|
||||
db.set_deadair(u, "f", Some(false)).await.unwrap();
|
||||
assert_eq!(page().await.enclosures[0].deadair, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_changes_to_the_catalogue_at_once_both_keep() {
|
||||
let ctx = Arc::new(test_ctx(config::Config::default()).await);
|
||||
|
||||
@@ -61,7 +61,11 @@ pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> {
|
||||
tracing::info!(feed = id, "forgot a failing feed nobody subscribes to");
|
||||
}
|
||||
|
||||
let candidates = db.reap_candidates().await?;
|
||||
let mut candidates = db.reap_candidates().await?;
|
||||
// A file's DeadAir copy is on the disk too, and goes when it goes (#171).
|
||||
for c in &mut candidates {
|
||||
c.bytes += copy_bytes(&c.path) as i64;
|
||||
}
|
||||
|
||||
if cfg.general.max_age_days > 0 {
|
||||
let cutoff = now() - (cfg.general.max_age_days * 86_400) as i64;
|
||||
@@ -100,7 +104,7 @@ async fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result<u64> {
|
||||
let size = std::fs::metadata(&c.path)
|
||||
.ok()
|
||||
.filter(|m| m.is_file())
|
||||
.map_or(c.bytes.max(0) as u64, |m| m.len());
|
||||
.map_or(c.bytes.max(0) as u64, |m| m.len() + copy_bytes(&c.path));
|
||||
// Missing is fine: the point is that it is gone.
|
||||
if let Err(e) = crate::download::remove(std::path::Path::new(&c.path))
|
||||
&& e.kind() != std::io::ErrorKind::NotFound
|
||||
@@ -112,6 +116,10 @@ async fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result<u64> {
|
||||
Ok(size)
|
||||
}
|
||||
|
||||
fn copy_bytes(path: &str) -> u64 {
|
||||
std::fs::metadata(crate::deadair::copy_path(std::path::Path::new(path))).map_or(0, |m| m.len())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
58
src/web.rs
58
src/web.rs
@@ -32,6 +32,7 @@ pub struct WebState {
|
||||
pub events: broadcast::Sender<Event>,
|
||||
/// Cloudflare Access's signing keys, fetched once and kept.
|
||||
pub access: Arc<crate::access::Keys>,
|
||||
pub playing: crate::deadair::Playing,
|
||||
}
|
||||
|
||||
pub fn router(state: WebState) -> Router {
|
||||
@@ -67,6 +68,7 @@ pub fn router(state: WebState) -> Router {
|
||||
.route("/admin", get(admin_page))
|
||||
.route("/admin.js", get(admin_js))
|
||||
.route("/media/{id}", get(media))
|
||||
.route("/media/{id}/deadair", get(media_deadair))
|
||||
.route("/api/art", get(art))
|
||||
.layer(middleware::from_fn_with_state(state.clone(), auth))
|
||||
// Signing in cannot require being signed in, so these sit outside the auth layer.
|
||||
@@ -373,8 +375,12 @@ async fn me(
|
||||
let url = state.ctx.cfg().web.sign_out_url.clone();
|
||||
let sign_out = (by_proxy && !url.is_empty()).then_some(url);
|
||||
let blocked = state.ctx.db.blocklist(user.id, "").await.unwrap_or_default();
|
||||
let prefs = state.ctx.db.deadair_prefs(Some(user.id)).await.unwrap_or_default();
|
||||
let deadair = prefs.iter().any(|(_, f, on)| f.is_empty() && *on);
|
||||
let saved = state.ctx.db.deadair_total(user.id).await.unwrap_or(0) / 1000;
|
||||
Json(serde_json::json!({
|
||||
"name": user.name, "admin": user.is_admin, "sign_out": sign_out, "blocked": blocked,
|
||||
"deadair": deadair, "deadair_saved": saved,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -382,6 +388,8 @@ async fn me(
|
||||
struct MePatch {
|
||||
/// Words that hide an item in every feed you read.
|
||||
blocked: Option<Vec<String>>,
|
||||
/// DeadAir's switch, beside the speed (#171).
|
||||
deadair: Option<bool>,
|
||||
}
|
||||
|
||||
/// Saves the block list for every feed. The theme is not the account's: each browser keeps its
|
||||
@@ -394,6 +402,9 @@ async fn patch_me(
|
||||
if let Some(words) = body.blocked {
|
||||
state.ctx.db.set_blocklist(user.id, "", &clean_words(words)?).await?;
|
||||
}
|
||||
if let Some(on) = body.deadair {
|
||||
state.ctx.db.set_deadair(user.id, "", Some(on)).await?;
|
||||
}
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
@@ -768,6 +779,9 @@ struct FeedRow {
|
||||
keywords: Vec<String>,
|
||||
/// Your words that hide an item of this feed, beside your list for every feed.
|
||||
blocked: Vec<String>,
|
||||
/// Played from DeadAir's copies when your switch is on: false for a feed you keep out, by
|
||||
/// itself or by its group.
|
||||
deadair: bool,
|
||||
allow_explicit: bool,
|
||||
auto_download: bool,
|
||||
max_new_per_check: Option<usize>,
|
||||
@@ -832,6 +846,8 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
|
||||
.collect();
|
||||
let counts = state.ctx.db.subscriber_counts().instrument(step("subscriber_counts")).await?;
|
||||
let pinned = state.ctx.db.pinned_feeds(user.id).instrument(step("pinned")).await?;
|
||||
let deadair: std::collections::HashMap<String, bool> =
|
||||
state.ctx.db.deadair_prefs(Some(user.id)).await?.into_iter().map(|(_, f, w)| (f, w)).collect();
|
||||
let mut listed = state.ctx.db.feed_list(user.id, only).instrument(step("feed_list")).await?;
|
||||
let mut out = Vec::with_capacity(mine.len());
|
||||
for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id)) {
|
||||
@@ -856,6 +872,7 @@ async fn feed_rows(state: &WebState, user: &crate::db::User, only: Option<&str>)
|
||||
.or_else(|| up.and_then(|u| u.keywords.clone()))
|
||||
.unwrap_or_else(|| feed.keywords.clone()),
|
||||
blocked,
|
||||
deadair: deadair.get(id).or_else(|| feed.group.as_deref().and_then(|g| deadair.get(g))).copied().unwrap_or(true),
|
||||
allow_explicit: mine
|
||||
.allow_explicit
|
||||
.or(up.and_then(|u| u.allow_explicit))
|
||||
@@ -1444,6 +1461,7 @@ async fn entry_page(
|
||||
row.description = Some(clean_description(&mut sanitizer, d, row.link.as_deref()));
|
||||
}
|
||||
}
|
||||
crate::deadair::for_reader(&state.ctx, user_id, &mut rows).await?;
|
||||
let total = db.count_in(user_id, feed, filter, search).await?;
|
||||
Ok(Json(EntryPage { total, entries: rows }))
|
||||
}
|
||||
@@ -1547,6 +1565,10 @@ struct FeedPatch {
|
||||
#[serde(default, deserialize_with = "double_option")]
|
||||
max_new_per_check: Option<Option<usize>>,
|
||||
pinned: Option<bool>,
|
||||
/// False keeps the feed out of DeadAir, true lets it in under a group kept out, null
|
||||
/// follows the group again.
|
||||
#[serde(default, deserialize_with = "double_option")]
|
||||
deadair: Option<Option<bool>>,
|
||||
}
|
||||
|
||||
/// A block list as sent: trimmed, blanks and repeats dropped, and bounded, since each word is
|
||||
@@ -1584,6 +1606,9 @@ async fn patch_feed(
|
||||
{
|
||||
return Err(ApiError::not_found("you do not subscribe to that feed"));
|
||||
}
|
||||
if let Some(wanted) = body.deadair {
|
||||
state.ctx.db.set_deadair(user.id, &id, wanted).await?;
|
||||
}
|
||||
// What one person wants -- which items, whether to fetch them, how many at a time --
|
||||
// is theirs. It goes on their subscription and nobody else sees the change.
|
||||
if let Some(mut mine) = state.ctx.db.subscription(user.id, &id).await? {
|
||||
@@ -1877,15 +1902,42 @@ async fn media(
|
||||
user: crate::db::User,
|
||||
req: Request,
|
||||
) -> Response {
|
||||
serve_media(state, id, user, req, false).await
|
||||
}
|
||||
|
||||
/// DeadAir's copy of a file (#171), at an address of its own: a player that had part of one
|
||||
/// version cached and asked the same address for more would stitch the two together.
|
||||
async fn media_deadair(
|
||||
State(state): State<WebState>,
|
||||
Path(id): Path<i64>,
|
||||
user: crate::db::User,
|
||||
req: Request,
|
||||
) -> Response {
|
||||
serve_media(state, id, user, req, true).await
|
||||
}
|
||||
|
||||
async fn serve_media(state: WebState, id: i64, user: crate::db::User, req: Request, copy: bool) -> Response {
|
||||
let Ok(Some(enc)) = state.ctx.db.enclosure(id).await else {
|
||||
return (StatusCode::NOT_FOUND, "no such enclosure").into_response();
|
||||
};
|
||||
if let Err(e) = subscribed(&state, &user, &enc.feed_id).await {
|
||||
return e.into_response();
|
||||
}
|
||||
let Some(path) = enc.path else {
|
||||
let Some(mut path) = enc.path else {
|
||||
return (StatusCode::NOT_FOUND, "not downloaded").into_response();
|
||||
};
|
||||
// Which version this person is playing, so the positions their player saves are read in its
|
||||
// clock (`deadair::on_save`).
|
||||
let key = (user.id, enc.feed_id.clone(), enc.guid.clone());
|
||||
if copy {
|
||||
let Some(c) = state.ctx.db.copies(&[id]).await.ok().and_then(|c| c.into_iter().next()) else {
|
||||
return (StatusCode::NOT_FOUND, "no DeadAir copy").into_response();
|
||||
};
|
||||
path = c.path.clone();
|
||||
state.playing.lock().unwrap().insert(key, Arc::new(c));
|
||||
} else {
|
||||
state.playing.lock().unwrap().remove(&key);
|
||||
}
|
||||
match ServeFile::new(crate::download::playable(std::path::Path::new(&path))).oneshot(req).await {
|
||||
Ok(r) => r.into_response(),
|
||||
Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(),
|
||||
@@ -1974,7 +2026,9 @@ async fn set_position(
|
||||
user: crate::db::User,
|
||||
Json(body): Json<Position>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
state.ctx.db.set_position(user.id, &feed_id, &guid, body.secs, body.duration).await?;
|
||||
let (secs, duration) =
|
||||
crate::deadair::on_save(&state.ctx, &state.playing, user.id, &feed_id, &guid, body.secs, body.duration).await?;
|
||||
state.ctx.db.set_position(user.id, &feed_id, &guid, secs, duration).await?;
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user