From 484aaa184902813d99ed99882df99e2dbdcc9c87 Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 18 Sep 2026 18:11:28 +0000 Subject: [PATCH 1/9] SeaORM beside rusqlite: entities, schema sync, a second connection The first step of moving to SeaORM (#18, phase 1). Nothing a user sees changes. - src/entity.rs: the seven tables as SeaORM entities, matching the SQLite schema. Strings are Text, as the columns are; yes/no columns are bool, which is BOOLEAN on Postgres and stays INTEGER in the existing SQLite file (sync notes the difference and leaves it alone). - Db holds a SeaORM connection to the same SQLite file beside the rusqlite one; functions move to it one at a time, and rusqlite goes with the last of them. - db::sync creates what a database is missing from the entities (SeaORM's schema-sync, experimental, so sea-orm is pinned to ~2.0), plus the two indexes an entity cannot express. Checked against a copy of production: it added the lower(name) index and changed nothing else. - Test databases are now built from the entities alone, in a temporary file (two connections to one ":memory:" are two databases), so every test also checks that the entities describe what the queries need. That caught the one difference: finding a user by name relied on COLLATE NOCASE, which Postgres lacks; it now compares lower() on both sides. - rusqlite steps back to 0.39: 0.40's libsqlite3-sys is newer than sqlx accepts, and only one may link SQLite. It goes away at the end of this phase. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 859 +++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 +- src/db.rs | 191 +++++++---- src/entity.rs | 224 ++++++++++++ src/main.rs | 29 +- src/retention.rs | 12 +- 6 files changed, 1213 insertions(+), 105 deletions(-) create mode 100644 src/entity.rs diff --git a/Cargo.lock b/Cargo.lock index 189bb90..ca082d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,7 +117,7 @@ checksum = "134c52ddac6d63c576bef8168db10c83c49c26444ecbc68060fef078925a901c" dependencies = [ "base64ct", "blake2", - "cpufeatures", + "cpufeatures 0.3.1", "password-hash", ] @@ -181,6 +181,15 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atoi" version = "3.1.0" @@ -344,6 +353,20 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -355,6 +378,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "bitvec" @@ -374,7 +400,16 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ - "digest", + "digest 0.11.3", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array 0.14.9", ] [[package]] @@ -386,6 +421,30 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "bs58" version = "0.5.1" @@ -455,7 +514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -512,7 +571,7 @@ version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 3.0.5", @@ -613,6 +672,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -622,6 +690,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.1" @@ -631,12 +714,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array 0.14.9", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -796,6 +898,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-where" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e2b94854e8576378ccda7c8de8a66ed8b4e8acbd2c50ec3418ea6c8aaf4b567" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -827,14 +940,46 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.6", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.12.1", + "crypto-common 0.2.2", "ctutils", ] @@ -891,6 +1036,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dtoa" version = "1.0.11" @@ -923,6 +1074,9 @@ name = "either" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" @@ -955,6 +1109,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -990,6 +1164,17 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1065,6 +1250,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.34" @@ -1126,6 +1322,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1274,19 +1480,22 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "foldhash", -] [[package]] name = "hashlink" -version = "0.12.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.17.1", + "hashbrown 0.16.1", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -1299,6 +1508,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "html5ever" version = "0.39.0" @@ -1622,6 +1849,7 @@ dependencies = [ "reqwest", "rss", "rusqlite", + "sea-orm", "serde", "serde_json", "tokio", @@ -1639,6 +1867,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.15.0" @@ -1806,6 +2043,12 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.23" @@ -1838,7 +2081,7 @@ dependencies = [ "hex", "http", "intervaltree", - "itertools", + "itertools 0.15.0", "librqbit-bencode", "librqbit-buffers", "librqbit-clone-to-owned", @@ -1853,7 +2096,7 @@ dependencies = [ "librqbit-utp", "memmap2", "mime_guess", - "nix", + "nix 0.30.1", "parking_lot", "rand 0.10.2", "regex", @@ -1887,7 +2130,7 @@ checksum = "be6410a7434e6c2b148931b6d109b57939cfed04cefa86451197c5f5198edf91" dependencies = [ "anyhow", "arrayvec", - "atoi", + "atoi 3.1.0", "bytes", "librqbit-buffers", "librqbit-clone-to-owned", @@ -1930,7 +2173,7 @@ dependencies = [ "directories", "encoding_rs", "hex", - "itertools", + "itertools 0.15.0", "librqbit-bencode", "librqbit-buffers", "librqbit-clone-to-owned", @@ -2002,7 +2245,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d1d2699a46a9dc83693c5fbbf288a17b77a24b9924815a69bfc97853a47af42" dependencies = [ "anyhow", - "atoi", + "atoi 3.1.0", "bstr", "futures", "httparse", @@ -2025,7 +2268,7 @@ dependencies = [ "bitvec", "byteorder", "bytes", - "itertools", + "itertools 0.15.0", "librqbit-bencode", "librqbit-buffers", "librqbit-clone-to-owned", @@ -2057,7 +2300,7 @@ dependencies = [ "backon", "byteorder", "futures", - "itertools", + "itertools 0.15.0", "librqbit-bencode", "librqbit-buffers", "librqbit-core", @@ -2122,9 +2365,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.38.2" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -2158,6 +2401,17 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "mac_address" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" +dependencies = [ + "nix 0.29.0", + "serde", + "winapi", +] + [[package]] name = "maplit" version = "1.0.2" @@ -2190,6 +2444,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2205,6 +2469,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.6" @@ -2297,6 +2570,19 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + [[package]] name = "nix" version = "0.30.1" @@ -2337,6 +2623,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.2.4" @@ -2427,6 +2723,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -2466,6 +2777,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pgvector" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3673cba5b9a124916096a423b806a9f29620972c6c97b08db5f2053e9428b481" +dependencies = [ + "serde", +] + [[package]] name = "phc" version = "0.6.1" @@ -2528,6 +2848,16 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "pluralizer" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3eba432a00a1f6c16f39147847a870e94e2e9b992759b503e330efec778cbe" +dependencies = [ + "once_cell", + "regex", +] + [[package]] name = "portable-atomic" version = "1.15.0" @@ -2573,6 +2903,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -2701,13 +3040,24 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -2722,6 +3072,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -2732,6 +3092,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -2943,9 +3312,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags 2.13.1", "fallible-iterator", @@ -2956,6 +3325,23 @@ dependencies = [ "sqlite-wasm-rs", ] +[[package]] +name = "rust_decimal" +version = "1.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand 0.8.8", + "rand 0.9.5", + "serde", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2979,6 +3365,7 @@ checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3106,6 +3493,129 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sea-bae" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "260bbc7148a8d6818ac5032a1b9970da1a68bdd723d45b12d59f7c1bf3565e24" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sea-orm" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01d46a6e22b8ce84aba64fe97011669859bc7f7120a47f5b58839dcaaa4545c" +dependencies = [ + "async-stream", + "async-trait", + "bigdecimal", + "chrono", + "derive-where", + "derive_more", + "futures-util", + "itertools 0.14.0", + "log", + "mac_address", + "pgvector", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-query-sqlx", + "sea-schema", + "serde", + "serde_json", + "sqlx", + "sqlx-core", + "strum", + "thiserror 2.0.20", + "time", + "tracing", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "sea-orm-macros" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e24c82fc1e76c014dffe5ecdac8f654da73161d703a16ea13414581e07459c83" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "pluralizer", + "proc-macro2", + "quote", + "sea-bae", + "syn 2.0.119", + "unicode-ident", +] + +[[package]] +name = "sea-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "546040c653a705e60ec65ecd3191a809603734bebbc225775916dea9ae409b31" +dependencies = [ + "ordered-float", + "sea-query-derive", + "serde_json", +] + +[[package]] +name = "sea-query-derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b0f466921cdd3cf4b89d5c3ac2173dba89a873ab395b123a645de181ec7537" +dependencies = [ + "darling 0.20.11", + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.119", + "thiserror 2.0.20", +] + +[[package]] +name = "sea-query-sqlx" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eaa419cdb9157da1361186b1959983eb2ea0dcb9a3c69dc45c449ecb2af8fef" +dependencies = [ + "sea-query", + "sqlx", +] + +[[package]] +name = "sea-schema" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3553c77dceed56e95bece9ea876c4dd67ca879ef51055a0b97a7bb89a8ae4fed" +dependencies = [ + "async-trait", + "sea-query", + "sea-query-sqlx", + "sea-schema-derive", + "sqlx", +] + +[[package]] +name = "sea-schema-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "debdc8729c37fdbf88472f97fd470393089f997a909e535ff67c544d18cfccf0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -3256,6 +3766,39 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -3315,7 +3858,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ed5f6ab2122c6dec69dca18c72fa4590a27e581ad20d44960fe74c032a0b23b" dependencies = [ - "generic-array", + "generic-array 0.12.4", "num", ] @@ -3330,6 +3873,9 @@ name = "smallvec" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -3341,6 +3887,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + [[package]] name = "spinning_top" version = "0.3.0" @@ -3362,6 +3917,178 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64 0.22.1", + "bytes", + "cfg-if", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.1", + "hashlink", + "indexmap 2.14.2", + "log", + "memchr", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.119", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" +dependencies = [ + "cfg-if", + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.119", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags 2.13.1", + "byteorder", + "bytes", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array 0.14.9", + "log", + "percent-encoding", + "serde", + "sha1", + "sha2 0.11.0", + "sqlx-core", + "thiserror 2.0.20", + "tracing", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi 2.0.0", + "base64 0.22.1", + "bitflags 2.13.1", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "rand 0.10.2", + "serde", + "serde_json", + "sha2 0.11.0", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.20", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi 2.0.0", + "flume", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "thiserror 2.0.20", + "tracing", + "url", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3392,12 +4119,29 @@ dependencies = [ "quote", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" + [[package]] name = "subtle" version = "2.6.1" @@ -3696,6 +4440,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime", + "toml_parser", + "winnow", +] + [[package]] name = "toml_parser" version = "1.1.3+spec-1.1.0" @@ -3873,12 +4629,39 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.7.1" @@ -3930,6 +4713,7 @@ checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" dependencies = [ "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -3945,6 +4729,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" @@ -3988,6 +4778,7 @@ dependencies = [ "cfg-if", "once_cell", "rustversion", + "serde", "wasm-bindgen-macro", "wasm-bindgen-shared", ] @@ -4088,6 +4879,21 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" + [[package]] name = "winapi" version = "0.3.9" @@ -4327,6 +5133,9 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen" diff --git a/Cargo.toml b/Cargo.toml index 8462ffc..847a9a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,8 @@ percent-encoding = "2.3.2" quick-xml = { version = "0.42.0", features = ["escape-html"] } reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "http2", "gzip", "stream", "json", "charset", "system-proxy"] } rss = "2.1.1" -rusqlite = { version = "0.40.2", features = ["bundled"] } +rusqlite = { version = "0.39", features = ["bundled"] } +sea-orm = { version = "~2.0.3", default-features = false, features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "schema-sync", "sqlite-use-returning-for-3_35"] } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "fs", "io-util", "net", "sync", "time", "signal"] } diff --git a/src/db.rs b/src/db.rs index e891c22..13ff5ca 100644 --- a/src/db.rs +++ b/src/db.rs @@ -7,8 +7,63 @@ use std::sync::Mutex; /// ponytail: one global connection mutex. Writes here are tiny and rare; move to a /// spawn_blocking pool if a large feed count ever makes it contend. +/// +/// Mid-way through moving to SeaORM (issue #18): `orm` is the new connection, to the same +/// SQLite file, and functions move to it one at a time; `conn` goes when the last one has. pub struct Db { conn: Mutex, + orm: sea_orm::DatabaseConnection, + /// A test's database file, removed when the test is done with it. + #[cfg(test)] + tmp: Option, +} + +#[cfg(test)] +impl Drop for Db { + fn drop(&mut self) { + if let Some(p) = &self.tmp { + for ext in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{ext}", p.display())); + } + } + } +} + +/// Creates whatever the database is missing from the entities in `crate::entity`: tables, +/// columns, unique keys, foreign keys. It only ever adds. What an entity cannot say -- an +/// index on an expression, or on two columns -- follows as plain SQL both databases accept. +/// +/// ponytail: schema sync is experimental in SeaORM 2 and exempt from semver, hence the +/// `~2.0` pin in Cargo.toml; if it changes shape, move to sea-orm-migration files. +async fn sync(orm: &sea_orm::DatabaseConnection) -> Result<()> { + use crate::entity::*; + use sea_orm::ConnectionTrait; + orm.get_schema_builder() + .register(feeds::Entity) + .register(entries::Entity) + .register(enclosures::Entity) + .register(users::Entity) + .register(subscriptions::Entity) + .register(entry_state::Entity) + .register(sessions::Entity) + .sync(orm) + .await + .context("creating the schema")?; + for sql in [ + "CREATE INDEX IF NOT EXISTS enclosures_entry ON enclosures (feed_id, guid)", + "CREATE UNIQUE INDEX IF NOT EXISTS users_name_lower ON users (lower(name))", + ] { + orm.execute_unprepared(sql).await.with_context(|| sql.to_owned())?; + } + Ok(()) +} + +async fn connect(path: &Path) -> Result { + let mut opts = sea_orm::ConnectOptions::new(format!("sqlite://{}?mode=rwc", path.display())); + opts.sqlx_logging(false); + sea_orm::Database::connect(opts) + .await + .with_context(|| format!("opening {}", path.display())) } const SCHEMA: &str = " @@ -243,7 +298,7 @@ pub struct FeedSummary { } impl Db { - pub fn open(path: &Path) -> Result { + pub async fn open(path: &Path) -> Result { if let Some(dir) = path.parent() { std::fs::create_dir_all(dir) .with_context(|| format!("creating {}", dir.display()))?; @@ -255,18 +310,33 @@ impl Db { conn.pragma_update(None, "busy_timeout", 5000)?; conn.execute_batch(SCHEMA).context("creating schema")?; migrate(&conn).context("migrating schema")?; - Ok(Self { conn: Mutex::new(conn) }) + let orm = connect(path).await?; + sync(&orm).await?; + Ok(Self { + conn: Mutex::new(conn), + orm, + #[cfg(test)] + tmp: None, + }) } - /// In-memory database, for tests. + /// A fresh database for a test, in a file of its own: two connections to one ":memory:" + /// would be two databases. Its schema comes from the entities alone (`sync`), not SCHEMA, + /// so every test also checks that the entities describe everything the queries need. #[cfg(test)] - pub fn memory() -> Result { - let conn = Connection::open_in_memory()?; - conn.execute_batch(SCHEMA)?; - // Same path as a real open, so a column added only in migrate() cannot pass the - // tests while being missing in production (or the reverse). - migrate(&conn)?; - Ok(Self { conn: Mutex::new(conn) }) + pub async fn memory() -> Result { + static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let path = std::env::temp_dir().join(format!( + "ipx-test-{}-{}.db", + std::process::id(), + N.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let orm = connect(&path).await?; + sync(&orm).await?; + let conn = Connection::open(&path)?; + conn.pragma_update(None, "foreign_keys", "ON")?; + conn.pragma_update(None, "busy_timeout", 5000)?; + Ok(Self { conn: Mutex::new(conn), orm, tmp: Some(path) }) } #[cfg(test)] @@ -1134,8 +1204,11 @@ impl Db { Ok(conn.last_insert_rowid()) } + /// Without regard to case, on either database: lower() both sides, which the unique index + /// on lower(name) serves. SQLite's COLLATE NOCASE on the column did this before, and + /// Postgres has no such thing. pub fn user_by_name(&self, name: &str) -> Result> { - self.one_user(&format!("SELECT {USER_COLS} FROM users WHERE name = ?1"), name) + self.one_user(&format!("SELECT {USER_COLS} FROM users WHERE lower(name) = lower(?1)"), name) } pub fn user_by_id(&self, id: i64) -> Result> { @@ -1594,9 +1667,9 @@ pub fn now() -> i64 { mod tests { use super::*; - #[test] - fn a_file_wordpress_listed_twice_is_folded_into_one() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn a_file_wordpress_listed_twice_is_folded_into_one() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'f','a','https://x/a.mp3','/d/a-2.mp3','done'), @@ -1652,9 +1725,9 @@ mod tests { assert_eq!(row(&conn).0.as_deref(), Some("e2")); } - #[test] - fn every_sort_column_runs_and_orders_both_ways() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn every_sort_column_runs_and_orders_both_ways() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1); INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g'); @@ -1697,9 +1770,9 @@ mod tests { assert!(!order_sql("x'; --", "asc", false).contains("x'")); } - #[test] - fn deleting_a_shared_file_asks_about_everyone_else() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn deleting_a_shared_file_asks_about_everyone_else() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0),(3,'kit',0); INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f'),(3,'f'); @@ -1723,9 +1796,9 @@ mod tests { assert_eq!(db.others_wanting(1, 3).unwrap(), (0, 0)); } - #[test] - fn read_state_belongs_to_one_person() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn read_state_belongs_to_one_person() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); INSERT INTO entries (feed_id, guid, title, first_seen) VALUES @@ -1760,9 +1833,9 @@ mod tests { assert_eq!(db.unread_count(1, "f").unwrap(), 1); } - #[test] - fn schema_is_idempotent_and_summary_handles_unknown_feeds() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn schema_is_idempotent_and_summary_handles_unknown_feeds() { + let db = Db::memory().await.unwrap(); // Re-running the schema must not fail: open() does this on every start. db.conn.lock().unwrap().execute_batch(SCHEMA).unwrap(); @@ -1826,9 +1899,9 @@ mod tests { assert_eq!(kept, 1); } - #[test] - fn a_renamed_account_keeps_everything_but_its_name() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn a_renamed_account_keeps_everything_but_its_name() { + let db = Db::memory().await.unwrap(); let ray = db.create_user("rays", None, true).unwrap(); db.create_user("sam", None, false).unwrap(); db.subscribe(ray, "f").unwrap(); @@ -1840,9 +1913,9 @@ mod tests { assert!(db.rename_user(ray, "sam").is_err(), "a taken name is refused"); } - #[test] - fn an_account_knows_when_it_was_made_and_last_signed_in() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn an_account_knows_when_it_was_made_and_last_signed_in() { + let db = Db::memory().await.unwrap(); let id = db.create_user("ray", None, true).unwrap(); let get = || db.user_by_id(id).unwrap().unwrap(); assert!(get().created.is_some_and(|t| t > 0)); @@ -1858,11 +1931,11 @@ mod tests { assert!(get().last_login.unwrap() >= first); } - #[test] - fn the_first_admin_starts_with_the_catalogue_and_only_once() { + #[tokio::test] + async fn the_first_admin_starts_with_the_catalogue_and_only_once() { // Cutting this along with the dead read columns left the browser suite's admin with an // empty sidebar: it is how a fresh install's first account gets config.toml's feeds. - let db = Db::memory().unwrap(); + let db = Db::memory().await.unwrap(); db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',1);") .unwrap(); let subs = || -> i64 { @@ -1877,12 +1950,12 @@ mod tests { assert_eq!(subs(), 1); } - #[test] - fn every_filter_works_with_and_without_a_search_term() { + #[tokio::test] + async fn every_filter_works_with_and_without_a_search_term() { // Regression: the search clause used to be omitted when no term was given, while // ?2 was still bound -- rusqlite rejects a parameter the statement never mentions, // so plain filtering failed with "Wrong number of parameters passed to query". - let db = Db::memory().unwrap(); + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO entries (feed_id, guid, title, description, first_seen, duration) VALUES ('f','a','Alpha dive','notes one', 100,NULL), @@ -1952,10 +2025,10 @@ mod tests { assert_eq!(listening(), ["h", "e"]); } - #[test] - fn pending_takes_the_latest_episodes_first() { + #[tokio::test] + async fn pending_takes_the_latest_episodes_first() { // A cap of 3 must mean the three newest, not the three recorded first. - let db = Db::memory().unwrap(); + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO entries (feed_id, guid, published, first_seen) VALUES ('f','old',100,100), ('f','mid',200,200), ('f','new',300,300); @@ -1969,9 +2042,9 @@ mod tests { assert_eq!(got, vec!["u-new", "u-mid"], "newest first, oldest left for later"); } - #[test] - fn a_restart_requeues_interrupted_downloads() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn a_restart_requeues_interrupted_downloads() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO enclosures (id, feed_id, guid, url, state, path) VALUES (1,'f','a','u1','downloading',NULL), @@ -1990,9 +2063,9 @@ mod tests { assert_eq!(state(4), "done"); } - #[test] - fn a_show_takes_over_what_its_creator_held() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn a_show_takes_over_what_its_creator_held() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1); INSERT INTO enclosures (id, feed_id, guid, url, state, path, last_error) VALUES @@ -2026,9 +2099,9 @@ mod tests { assert!(db.skipped_by_filter("other").unwrap().is_empty(), "torrents disabled is not a filter"); } - #[test] - fn a_feed_in_a_group_follows_your_settings_on_the_group() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn a_feed_in_a_group_follows_your_settings_on_the_group() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); INSERT INTO subscriptions (user_id, feed_id, allow_explicit) VALUES @@ -2045,9 +2118,9 @@ mod tests { assert_eq!(explicit(None), [None, Some(false)], "outside a group nothing is inherited"); } - #[test] - fn error_since_marks_the_start_of_a_run_of_failures_and_clears_on_success() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn error_since_marks_the_start_of_a_run_of_failures_and_clears_on_success() { + let db = Db::memory().await.unwrap(); db.set_feed_error("f", "http://x", "HTTP 404").unwrap(); // Backdate it, as if this feed had already been failing a while, so a second // failure landing "now" is distinguishable from the first. @@ -2065,9 +2138,9 @@ mod tests { assert_eq!(after.error_since, None, "a clean check ends the run of failures"); } - #[test] - fn a_pin_survives_saving_the_feeds_settings_and_needs_a_subscription() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn a_pin_survives_saving_the_feeds_settings_and_needs_a_subscription() { + let db = Db::memory().await.unwrap(); let me = db.create_user("pat", None, false).unwrap(); assert!(!db.set_pinned(me, "f", true).unwrap(), "not subscribed: nothing to pin"); db.subscribe(me, "f").unwrap(); @@ -2080,9 +2153,9 @@ mod tests { assert!(db.pinned_feeds(me).unwrap().is_empty()); } - #[test] - fn enclosure_url_is_the_dedupe_key() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn enclosure_url_is_the_dedupe_key() { + let db = Db::memory().await.unwrap(); let conn = db.conn.lock().unwrap(); let insert = "INSERT INTO enclosures (feed_id, guid, url, state) VALUES ('f', 'g', 'http://x/a.mp3', 'pending')"; conn.execute(insert, []).unwrap(); diff --git a/src/entity.rs b/src/entity.rs new file mode 100644 index 0000000..f8a05cb --- /dev/null +++ b/src/entity.rs @@ -0,0 +1,224 @@ +//! The database's tables as SeaORM entities: the one description of the schema, from which +//! `Db::open` creates what a database is missing, on SQLite or Postgres alike (see `db::sync`). +//! What each column means is in the comments on `db::SCHEMA`, the SQLite schema these match. + +pub mod feeds { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "feeds")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub id: String, + #[sea_orm(column_type = "Text")] + pub url: String, + #[sea_orm(column_type = "Text", nullable)] + pub title: Option, + #[sea_orm(column_type = "Text", nullable)] + pub image: Option, + #[sea_orm(column_type = "Text", nullable)] + pub category: Option, + #[sea_orm(column_type = "Text", nullable)] + pub etag: Option, + #[sea_orm(column_type = "Text", nullable)] + pub last_modified: Option, + pub last_checked: Option, + pub ttl_mins: Option, + #[sea_orm(column_type = "Text", nullable)] + pub last_error: Option, + pub error_since: Option, + #[sea_orm(default_value = false)] + pub orphaned: bool, + #[sea_orm(column_type = "Text", nullable)] + pub group_id: Option, + #[sea_orm(default_value = false)] + pub managed: bool, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} +} + +pub mod entries { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "entries")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub feed_id: String, + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub guid: String, + #[sea_orm(column_type = "Text", nullable)] + pub title: Option, + #[sea_orm(column_type = "Text", nullable)] + pub link: Option, + pub published: Option, + #[sea_orm(column_type = "Text", nullable)] + pub description: Option, + pub first_seen: i64, + #[sea_orm(column_type = "Text", nullable)] + pub image: Option, + pub duration: Option, + pub episode: Option, + pub season: Option, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} +} + +pub mod enclosures { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "enclosures")] + pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + #[sea_orm(column_type = "Text")] + pub feed_id: String, + #[sea_orm(column_type = "Text")] + pub guid: String, + /// The dedupe key: one file serves every subscriber. + #[sea_orm(unique, column_type = "Text")] + pub url: String, + #[sea_orm(column_type = "Text", nullable)] + pub mime: Option, + pub length: Option, + #[sea_orm(column_type = "Text", nullable)] + pub path: Option, + #[sea_orm(column_type = "Text")] + pub state: String, + #[sea_orm(default_value = 0)] + pub bytes_done: i64, + pub downloaded_at: Option, + #[sea_orm(column_type = "Text", nullable)] + pub last_error: Option, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} +} + +pub mod users { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "users")] + pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + /// Unique without regard to case: `db::sync` adds the index on lower(name), which + /// works the same on both databases where SQLite's COLLATE NOCASE does not. + #[sea_orm(column_type = "Text")] + pub name: String, + #[sea_orm(column_type = "Text", nullable)] + pub pass_hash: Option, + #[sea_orm(default_value = false)] + pub is_admin: bool, + pub created: Option, + pub last_login: Option, + #[sea_orm(column_type = "Text", nullable)] + pub theme: Option, + #[sea_orm(column_type = "Text", nullable)] + pub theme_mode: Option, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} +} + +/// A table of one person's rows, gone when they are. +macro_rules! owned_by_user { + () => { + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation { + #[sea_orm( + belongs_to = "super::users::Entity", + from = "Column::UserId", + to = "super::users::Column::Id", + on_delete = "Cascade" + )] + User, + } + + impl Related for Entity { + fn to() -> RelationDef { + Relation::User.def() + } + } + + impl ActiveModelBehavior for ActiveModel {} + }; +} + +pub mod subscriptions { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "subscriptions")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub user_id: i64, + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub feed_id: String, + /// JSON array of strings; NULL follows the feed. + #[sea_orm(column_type = "Text", nullable)] + pub keywords: Option, + pub auto_download: Option, + pub allow_explicit: Option, + pub max_new_per_check: Option, + #[sea_orm(default_value = false)] + pub pinned: bool, + } + + owned_by_user!(); +} + +pub mod entry_state { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "entry_state")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub user_id: i64, + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub feed_id: String, + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub guid: String, + #[sea_orm(default_value = false)] + pub read: bool, + #[sea_orm(default_value = false)] + pub flagged: bool, + #[sea_orm(default_value = 0)] + pub position: i64, + pub duration: Option, + } + + owned_by_user!(); +} + +pub mod sessions { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)] + #[sea_orm(table_name = "sessions")] + pub struct Model { + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub token: String, + pub user_id: i64, + pub seen: i64, + } + + owned_by_user!(); +} diff --git a/src/main.rs b/src/main.rs index 787a4a5..134571d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod auth; mod config; mod db; +mod entity; mod download; mod feed; mod ipc; @@ -179,7 +180,7 @@ async fn main() -> Result<()> { let config_path = cli.config.clone().unwrap_or_else(config::config_path); let cfg = config::Config::load(&config_path)?; - let db = db::Db::open(&config::data_dir().join("state.db"))?; + let db = db::Db::open(&config::data_dir().join("state.db")).await?; // A daemon owns the state; don't have two processes downloading the same thing. let wire_cmd = match &cli.command { @@ -1690,8 +1691,8 @@ mod tests { } } - #[test] - fn a_shared_feed_is_fetched_for_whoever_wants_the_most() { + #[tokio::test] + async fn a_shared_feed_is_fetched_for_whoever_wants_the_most() { // Nobody subscribed: the feed's own settings stand, as in a single-user install. let p = merge_policy(&[], &feed(), 3); assert!(p.auto_download); @@ -1721,10 +1722,10 @@ mod tests { assert!(p.auto_download); } - fn test_ctx(cfg: config::Config) -> Ctx { + async fn test_ctx(cfg: config::Config) -> Ctx { Ctx { cfg: std::sync::RwLock::new(Arc::new(cfg)), - db: db::Db::memory().unwrap(), + db: db::Db::memory().await.unwrap(), client: reqwest::Client::new(), out: Emitter::terminal(), torrents: tokio::sync::OnceCell::new(), @@ -1734,11 +1735,11 @@ mod tests { } } - #[test] - fn a_derived_feed_is_not_scanned_once_its_opml_leaves_config() { + #[tokio::test] + async fn a_derived_feed_is_not_scanned_once_its_opml_leaves_config() { // davewiner: the OPML subscription left config.toml, but its 922 derived rows // stayed in the database and kept being scanned under the no-parent fallback. - let ctx = test_ctx(config::Config::default()); + let ctx = test_ctx(config::Config::default()).await; ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").unwrap(); assert!( subscriptions(&ctx).unwrap().iter().all(|s| s.id != "child"), @@ -1746,9 +1747,9 @@ mod tests { ); } - #[test] - fn retiring_a_group_drops_what_was_never_downloaded_and_orphans_the_rest() { - let ctx = test_ctx(config::Config::default()); + #[tokio::test] + async fn retiring_a_group_drops_what_was_never_downloaded_and_orphans_the_rest() { + let ctx = test_ctx(config::Config::default()).await; ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").unwrap(); ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").unwrap(); let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None }; @@ -1763,14 +1764,14 @@ mod tests { assert!(ctx.db.feed_summary("has-file").unwrap().orphaned, "and flagged as orphaned"); } - #[test] - fn a_stranded_group_is_retired_but_a_promoted_feed_keeps_its_entries() { + #[tokio::test] + async fn a_stranded_group_is_retired_but_a_promoted_feed_keeps_its_entries() { // davewiner: the OPML left config before retire_group existed, and 11 of its feeds // promoted to config since still said managed = 1. let mut cfg = config::Config::default(); cfg.feeds.insert("promoted".into(), feed()); cfg.feeds.insert("live-opml".into(), feed()); - let ctx = test_ctx(cfg); + let ctx = test_ctx(cfg).await; ctx.db.upsert_managed("promoted", "http://x/p.xml", "Promoted", "gone-opml").unwrap(); ctx.db.upsert_managed("empty", "http://x/e.xml", "Empty", "gone-opml").unwrap(); ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").unwrap(); diff --git a/src/retention.rs b/src/retention.rs index 232288a..66ecf5b 100644 --- a/src/retention.rs +++ b/src/retention.rs @@ -149,10 +149,10 @@ mod tests { // age_key 0 means "never recorded" -- not the same as "infinitely old". } - #[test] - fn query_never_offers_a_file_anyone_starred_and_prefers_ones_everyone_read() { + #[tokio::test] + async fn query_never_offers_a_file_anyone_starred_and_prefers_ones_everyone_read() { // One file serves both subscribers, so it takes both of them to release it. - let db = Db::memory().unwrap(); + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f'); @@ -185,9 +185,9 @@ mod tests { ); } - #[test] - fn prune_keeps_entries_that_still_have_a_file() { - let db = Db::memory().unwrap(); + #[tokio::test] + async fn prune_keeps_entries_that_still_have_a_file() { + let db = Db::memory().await.unwrap(); db.exec_for_test( "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1); INSERT INTO entry_state (user_id, feed_id, guid, flagged) VALUES (1,'f','flagged',1); From 4927677e6653074e8a0965dbc33f56f18530fe26 Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 18 Sep 2026 18:17:52 +0000 Subject: [PATCH 2/9] SeaORM: accounts, sessions and themes The fourteen user and session functions move from rusqlite to SeaORM and become async; their callers await them (auth, admin_user, user_cmd, the account handlers). Checked against a copy of production, where the yes/no columns are still INTEGER: the admin flag reads back right. Co-Authored-By: Claude Opus 5 --- src/db.rs | 249 +++++++++++++++++++++++++++------------------------- src/main.rs | 36 ++++---- src/web.rs | 56 ++++++------ 3 files changed, 177 insertions(+), 164 deletions(-) diff --git a/src/db.rs b/src/db.rs index 13ff5ca..9fbb9bc 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,7 +1,10 @@ //! SQLite state. Replaces the per-feed .ipxd plists, history.dat and qmcache.dat. use anyhow::{Context, Result}; +use crate::entity::{sessions, users}; use rusqlite::{Connection, OptionalExtension, params}; +use sea_orm::sea_query::{Expr, Func}; +use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, QueryOrder, Set}; use std::path::Path; use std::sync::Mutex; @@ -200,18 +203,17 @@ pub struct User { pub last_login: Option, } -/// The columns `user_row` reads, in its order. -const USER_COLS: &str = "id, name, pass_hash, is_admin, created, last_login"; - -fn user_row(r: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(User { - id: r.get(0)?, - name: r.get(1)?, - pass_hash: r.get(2)?, - is_admin: r.get::<_, i64>(3)? != 0, - created: r.get(4)?, - last_login: r.get(5)?, - }) +impl From for User { + fn from(u: users::Model) -> Self { + User { + id: u.id, + name: u.name, + pass_hash: u.pass_hash, + is_admin: u.is_admin, + created: u.created, + last_login: u.last_login, + } + } } /// A feed derived from an OPML subscription rather than written into the config. @@ -1195,141 +1197,152 @@ impl Db { // ---- users and sessions ---- - pub fn create_user(&self, name: &str, pass_hash: Option<&str>, admin: bool) -> Result { - let conn = self.conn.lock().unwrap(); - conn.execute( - "INSERT INTO users (name, pass_hash, is_admin, created) VALUES (?1, ?2, ?3, ?4)", - params![name, pass_hash, admin as i64, now()], - )?; - Ok(conn.last_insert_rowid()) + pub async fn create_user(&self, name: &str, pass_hash: Option<&str>, admin: bool) -> Result { + let m = users::ActiveModel { + name: Set(name.to_owned()), + pass_hash: Set(pass_hash.map(str::to_owned)), + is_admin: Set(admin), + created: Set(Some(now())), + ..Default::default() + } + .insert(&self.orm) + .await?; + Ok(m.id) } /// Without regard to case, on either database: lower() both sides, which the unique index /// on lower(name) serves. SQLite's COLLATE NOCASE on the column did this before, and /// Postgres has no such thing. - pub fn user_by_name(&self, name: &str) -> Result> { - self.one_user(&format!("SELECT {USER_COLS} FROM users WHERE lower(name) = lower(?1)"), name) + pub async fn user_by_name(&self, name: &str) -> Result> { + use sea_orm::sea_query::ExprTrait; + Ok(users::Entity::find() + .filter(Expr::expr(Func::lower(Expr::col(users::Column::Name))).eq(Func::lower(name))) + .one(&self.orm) + .await? + .map(User::from)) } - pub fn user_by_id(&self, id: i64) -> Result> { - self.one_user(&format!("SELECT {USER_COLS} FROM users WHERE id = ?1"), id) + pub async fn user_by_id(&self, id: i64) -> Result> { + Ok(users::Entity::find_by_id(id).one(&self.orm).await?.map(User::from)) } - fn one_user(&self, sql: &str, key: P) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare(sql)?; - let mut rows = stmt.query(params![key])?; - Ok(match rows.next()? { - Some(r) => Some(user_row(r)?), - None => None, - }) + pub async fn users(&self) -> Result> { + Ok(users::Entity::find() + .order_by_asc(users::Column::Name) + .all(&self.orm) + .await? + .into_iter() + .map(User::from) + .collect()) } - pub fn users(&self) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = - conn.prepare(&format!("SELECT {USER_COLS} FROM users ORDER BY name"))?; - let out = stmt - .query_map([], user_row)? - .collect::>>()?; - Ok(out) - } - - pub fn set_password(&self, id: i64, hash: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("UPDATE users SET pass_hash = ?2 WHERE id = ?1", params![id, hash])?; + /// Sets some of one person's columns, whatever else is in their row. + async fn update_user(&self, id: i64, m: users::ActiveModel) -> Result<()> { + users::Entity::update_many() + .set(m) + .filter(users::Column::Id.eq(id)) + .exec(&self.orm) + .await?; Ok(()) } - pub fn set_admin(&self, id: i64, admin: bool) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("UPDATE users SET is_admin = ?2 WHERE id = ?1", params![id, admin as i64])?; - Ok(()) + pub async fn set_password(&self, id: i64, hash: &str) -> Result<()> { + self.update_user(id, users::ActiveModel { pass_hash: Set(Some(hash.to_owned())), ..Default::default() }) + .await + } + + pub async fn set_admin(&self, id: i64, admin: bool) -> Result<()> { + self.update_user(id, users::ActiveModel { is_admin: Set(admin), ..Default::default() }).await } /// The proxy signs people in by the name it vouches for, so an account made before the proxy - /// was set up has to take that name to be found by it. The name is UNIQUE, so a taken one is + /// was set up has to take that name to be found by it. The name is unique, so a taken one is /// refused here as well as by the caller. - pub fn rename_user(&self, id: i64, name: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("UPDATE users SET name = ?2 WHERE id = ?1", params![id, name])?; - Ok(()) + pub async fn rename_user(&self, id: i64, name: &str) -> Result<()> { + self.update_user(id, users::ActiveModel { name: Set(name.to_owned()), ..Default::default() }).await } /// Records a sign-in, to the hour: the proxy vouches for every request, and writing each one /// would buy nothing. - pub fn signed_in(&self, id: i64) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE users SET last_login = ?2 WHERE id = ?1 AND coalesce(last_login, 0) <= ?2 - 3600", - params![id, now()], - )?; + pub async fn signed_in(&self, id: i64) -> Result<()> { + // Here only: it is implemented for every type, and file-wide it shadows i64::max. + use sea_orm::sea_query::ExprTrait; + let now = now(); + users::Entity::update_many() + .col_expr(users::Column::LastLogin, Expr::val(now).into()) + .filter(users::Column::Id.eq(id)) + .filter(Expr::expr(Func::coalesce([Expr::col(users::Column::LastLogin), Expr::val(0)])).lte(now - 3600)) + .exec(&self.orm) + .await?; Ok(()) } /// Sessions go with the user: a deleted account must not leave a usable cookie behind. - pub fn delete_user(&self, id: i64) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("DELETE FROM sessions WHERE user_id = ?1", [id])?; - conn.execute("DELETE FROM users WHERE id = ?1", [id])?; + /// The foreign key would take them anyway; this does not rely on it being switched on. + pub async fn delete_user(&self, id: i64) -> Result<()> { + sessions::Entity::delete_many().filter(sessions::Column::UserId.eq(id)).exec(&self.orm).await?; + users::Entity::delete_by_id(id).exec(&self.orm).await?; Ok(()) } - pub fn create_session(&self, user_id: i64, token: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "INSERT INTO sessions (token, user_id, seen) VALUES (?1, ?2, ?3)", - params![token, user_id, now()], - )?; + pub async fn create_session(&self, user_id: i64, token: &str) -> Result<()> { + sessions::ActiveModel { token: Set(token.to_owned()), user_id: Set(user_id), seen: Set(now()) } + .insert(&self.orm) + .await?; Ok(()) } /// The theme this person chose, and light, dark or auto; None for either until they choose. - pub fn theme(&self, user_id: i64) -> Result<(Option, Option)> { - let conn = self.conn.lock().unwrap(); - Ok(conn.query_row("SELECT theme, theme_mode FROM users WHERE id = ?1", [user_id], |r| { - Ok((r.get(0)?, r.get(1)?)) - })?) + pub async fn theme(&self, user_id: i64) -> Result<(Option, Option)> { + Ok(users::Entity::find_by_id(user_id) + .one(&self.orm) + .await? + .map(|u| (u.theme, u.theme_mode)) + .unwrap_or_default()) } - pub fn set_theme(&self, user_id: i64, theme: &str, mode: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE users SET theme = ?2, theme_mode = ?3 WHERE id = ?1", - params![user_id, theme, mode], - )?; - Ok(()) + pub async fn set_theme(&self, user_id: i64, theme: &str, mode: &str) -> Result<()> { + self.update_user( + user_id, + users::ActiveModel { + theme: Set(Some(theme.to_owned())), + theme_mode: Set(Some(mode.to_owned())), + ..Default::default() + }, + ) + .await } /// The user behind a session cookie, if it is still live. Idle sessions expire after /// `max_idle_secs`; touching `seen` is what keeps a session in daily use alive. - pub fn session_user(&self, token: &str, max_idle_secs: i64) -> Result> { - let conn = self.conn.lock().unwrap(); + pub async fn session_user(&self, token: &str, max_idle_secs: i64) -> Result> { + // Here only: it is implemented for every type, and file-wide it shadows i64::max. + use sea_orm::sea_query::ExprTrait; let cutoff = now() - max_idle_secs; - let mut stmt = conn.prepare( - "SELECT u.id, u.name, u.pass_hash, u.is_admin, u.created, u.last_login - FROM sessions s JOIN users u ON u.id = s.user_id - WHERE s.token = ?1 AND s.seen >= ?2", - )?; - let mut rows = stmt.query(params![token, cutoff])?; - let found = match rows.next()? { - Some(r) => Some(user_row(r)?), - None => None, - }; - drop(rows); - drop(stmt); + let found = sessions::Entity::find_by_id(token.to_owned()) + .filter(sessions::Column::Seen.gte(cutoff)) + .find_also_related(users::Entity) + .one(&self.orm) + .await? + .and_then(|(_, u)| u); if found.is_some() { - conn.execute("UPDATE sessions SET seen = ?2 WHERE token = ?1", params![token, now()])?; + sessions::Entity::update_many() + .col_expr(sessions::Column::Seen, Expr::val(now()).into()) + .filter(sessions::Column::Token.eq(token)) + .exec(&self.orm) + .await?; } else { // Either unknown or timed out; either way it is dead weight. - conn.execute("DELETE FROM sessions WHERE token = ?1 OR seen < ?2", params![token, cutoff])?; + sessions::Entity::delete_many() + .filter(sessions::Column::Token.eq(token).or(sessions::Column::Seen.lt(cutoff))) + .exec(&self.orm) + .await?; } - Ok(found) + Ok(found.map(User::from)) } - pub fn delete_session(&self, token: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("DELETE FROM sessions WHERE token = ?1", [token])?; + pub async fn delete_session(&self, token: &str) -> Result<()> { + sessions::Entity::delete_by_id(token.to_owned()).exec(&self.orm).await?; Ok(()) } @@ -1902,33 +1915,33 @@ mod tests { #[tokio::test] async fn a_renamed_account_keeps_everything_but_its_name() { let db = Db::memory().await.unwrap(); - let ray = db.create_user("rays", None, true).unwrap(); - db.create_user("sam", None, false).unwrap(); + let ray = db.create_user("rays", None, true).await.unwrap(); + db.create_user("sam", None, false).await.unwrap(); db.subscribe(ray, "f").unwrap(); - db.rename_user(ray, "rays@sdf1.net").unwrap(); - assert!(db.user_by_name("rays").unwrap().is_none()); - let renamed = db.user_by_name("RAYS@sdf1.net").unwrap().unwrap(); + db.rename_user(ray, "rays@sdf1.net").await.unwrap(); + assert!(db.user_by_name("rays").await.unwrap().is_none()); + let renamed = db.user_by_name("RAYS@sdf1.net").await.unwrap().unwrap(); assert_eq!((renamed.id, renamed.is_admin), (ray, true), "same account, still the admin"); assert_eq!(db.subscriptions_for(ray).unwrap().len(), 1, "and still subscribed"); - assert!(db.rename_user(ray, "sam").is_err(), "a taken name is refused"); + assert!(db.rename_user(ray, "sam").await.is_err(), "a taken name is refused"); } #[tokio::test] async fn an_account_knows_when_it_was_made_and_last_signed_in() { let db = Db::memory().await.unwrap(); - let id = db.create_user("ray", None, true).unwrap(); - let get = || db.user_by_id(id).unwrap().unwrap(); - assert!(get().created.is_some_and(|t| t > 0)); - assert_eq!(get().last_login, None, "made, but never signed in"); - db.signed_in(id).unwrap(); - let first = get().last_login.unwrap(); + let id = db.create_user("ray", None, true).await.unwrap(); + let get = async || db.user_by_id(id).await.unwrap().unwrap(); + assert!(get().await.created.is_some_and(|t| t > 0)); + assert_eq!(get().await.last_login, None, "made, but never signed in"); + db.signed_in(id).await.unwrap(); + let first = get().await.last_login.unwrap(); // Within the hour, the proxy vouching again writes nothing; after it, it does. db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 60)).unwrap(); - db.signed_in(id).unwrap(); - assert_eq!(get().last_login, Some(first - 60)); + db.signed_in(id).await.unwrap(); + assert_eq!(get().await.last_login, Some(first - 60)); db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 7200)).unwrap(); - db.signed_in(id).unwrap(); - assert!(get().last_login.unwrap() >= first); + db.signed_in(id).await.unwrap(); + assert!(get().await.last_login.unwrap() >= first); } #[tokio::test] @@ -2141,7 +2154,7 @@ mod tests { #[tokio::test] async fn a_pin_survives_saving_the_feeds_settings_and_needs_a_subscription() { let db = Db::memory().await.unwrap(); - let me = db.create_user("pat", None, false).unwrap(); + let me = db.create_user("pat", None, false).await.unwrap(); assert!(!db.set_pinned(me, "f", true).unwrap(), "not subscribed: nothing to pin"); db.subscribe(me, "f").unwrap(); assert!(db.set_pinned(me, "f", true).unwrap()); diff --git a/src/main.rs b/src/main.rs index 134571d..764a4a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -226,7 +226,7 @@ async fn main() -> Result<()> { add(&ctx, &config_path, &url, folder, keywords).await } Command::Rm { feed } => rm(&ctx, &config_path, &feed), - Command::User { cmd } => user_cmd(&ctx, cmd), + Command::User { cmd } => user_cmd(&ctx, cmd).await, Command::Import { file } => import(&ctx, &config_path, &file).await, Command::Export { file } => export(&ctx, &file), _ => run(&ctx, wire_cmd.expect("only List and Daemon have no wire form")).await, @@ -235,7 +235,7 @@ async fn main() -> Result<()> { /// Accounts. Passwords come in on stdin so they never reach a shell history or a `ps` /// listing. -fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { +async fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { let read_password = || -> Result { use std::io::Read; let mut buf = String::new(); @@ -253,7 +253,7 @@ fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { if name.is_empty() { anyhow::bail!("a name is required"); } - if ctx.db.user_by_name(&name)?.is_some() { + if ctx.db.user_by_name(&name).await?.is_some() { anyhow::bail!("{name} already exists"); } let hash = if no_password { @@ -262,8 +262,8 @@ fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { Some(crate::auth::hash_password(&read_password()?)?) }; // The first account runs the place; there is nobody else to grant it. - let first = ctx.db.users()?.is_empty(); - ctx.db.create_user(&name, hash.as_deref(), admin || first)?; + let first = ctx.db.users().await?.is_empty(); + ctx.db.create_user(&name, hash.as_deref(), admin || first).await?; println!( "added {name}{}{}", if admin || first { " (admin)" } else { "" }, @@ -272,7 +272,7 @@ fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { Ok(()) } UserCmd::List => { - let users = ctx.db.users()?; + let users = ctx.db.users().await?; if users.is_empty() { println!("no accounts yet: ipx user add "); } @@ -295,9 +295,9 @@ fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { let name = name.trim().to_ascii_lowercase(); let user = ctx .db - .user_by_name(&name)? + .user_by_name(&name).await? .ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?; - ctx.db.set_password(user.id, &crate::auth::hash_password(&read_password()?)?)?; + ctx.db.set_password(user.id, &crate::auth::hash_password(&read_password()?)?).await?; println!("password changed for {name}"); Ok(()) } @@ -308,12 +308,12 @@ fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { .ok_or_else(|| anyhow::anyhow!("not a usable name: no commas, semicolons or line breaks"))?; let user = ctx .db - .user_by_name(&name)? + .user_by_name(&name).await? .ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?; - if ctx.db.user_by_name(&new_name)?.is_some() { + if ctx.db.user_by_name(&new_name).await?.is_some() { anyhow::bail!("{new_name} already exists"); } - ctx.db.rename_user(user.id, &new_name)?; + ctx.db.rename_user(user.id, &new_name).await?; println!("renamed {name} to {new_name}"); Ok(()) } @@ -321,9 +321,9 @@ fn user_cmd(ctx: &Arc, cmd: UserCmd) -> Result<()> { let name = name.trim().to_ascii_lowercase(); let user = ctx .db - .user_by_name(&name)? + .user_by_name(&name).await? .ok_or_else(|| anyhow::anyhow!("no such account: {name}"))?; - ctx.db.delete_user(user.id)?; + ctx.db.delete_user(user.id).await?; println!("removed {name}"); Ok(()) } @@ -370,15 +370,15 @@ async fn daemon( } // A database with nobody in it cannot be signed into. - if ctx.db.users()?.is_empty() { - ctx.db.create_user("admin", Some(&crate::auth::hash_password(DEFAULT_PASSWORD)?), true)?; + if ctx.db.users().await?.is_empty() { + ctx.db.create_user("admin", Some(&crate::auth::hash_password(DEFAULT_PASSWORD)?), true).await?; tracing::warn!( "no accounts yet: created 'admin' with the default password '{DEFAULT_PASSWORD}'. \ Change it with `echo -n | ipx user passwd admin`" ); } - if let Some(admin) = ctx.db.users()?.into_iter().find(|u| u.is_admin) { + if let Some(admin) = ctx.db.users().await?.into_iter().find(|u| u.is_admin) { let catalogue: Vec = ctx.cfg().feeds.keys().cloned().collect(); match ctx.db.adopt_catalogue(admin.id, &catalogue) { Ok(0) => {} @@ -670,7 +670,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path // The CLI speaks for the operator, as the shared web token does. let admin = ctx .db - .users()? + .users().await? .into_iter() .find(|u| u.is_admin) .ok_or_else(|| anyhow::anyhow!("no admin account to subscribe: ipx user add --admin"))?; @@ -1317,7 +1317,7 @@ async fn sync_group( .map(|m| m.id.clone()) .chain(std::iter::once(parent_id.to_string())) { - for user in ctx.db.users()? { + for user in ctx.db.users().await? { if ctx.db.subscription(user.id, parent_id)?.is_some() { ctx.db.subscribe(user.id, &id)?; } diff --git a/src/web.rs b/src/web.rs index b40f03b..5c4fd72 100644 --- a/src/web.rs +++ b/src/web.rs @@ -107,16 +107,16 @@ async fn auth(State(state): State, mut req: Request, next: Next) -> Re let mut user = None; if let Some(name) = vouched { - user = match state.ctx.db.user_by_name(&name) { + user = match state.ctx.db.user_by_name(&name).await { Ok(Some(u)) => Some(u), Ok(None) if cfg.web.auto_create_users => { tracing::info!(user = %name, "creating an account for a name the proxy vouched for"); - state - .ctx - .db - .create_user(&name, None, state.ctx.db.users().map(|u| u.is_empty()).unwrap_or(false)) - .ok() - .and_then(|id| state.ctx.db.user_by_id(id).ok().flatten()) + // The first account made is the admin. + let first = state.ctx.db.users().await.map(|u| u.is_empty()).unwrap_or(false); + match state.ctx.db.create_user(&name, None, first).await { + Ok(id) => state.ctx.db.user_by_id(id).await.ok().flatten(), + Err(_) => None, + } } Ok(None) => { tracing::warn!(user = %name, "proxy vouched for an unknown name and auto_create_users is off"); @@ -130,7 +130,7 @@ async fn auth(State(state): State, mut req: Request, next: Next) -> Re // Every request comes vouched for; signed_in keeps one an hour. Failing to note the time // must not turn anyone away, so its error goes unanswered. if let Some(u) = &user { - let _ = state.ctx.db.signed_in(u.id); + let _ = state.ctx.db.signed_in(u.id).await; } } let by_proxy = user.is_some(); @@ -141,7 +141,7 @@ async fn auth(State(state): State, mut req: Request, next: Next) -> Re user = state .ctx .db - .session_user(&sid, cfg.web.session_days.max(1) * 86_400) + .session_user(&sid, cfg.web.session_days.max(1) * 86_400).await .unwrap_or(None); } } @@ -155,11 +155,11 @@ async fn auth(State(state): State, mut req: Request, next: Next) -> Re if user.is_none() && !token.is_empty() { let supplied = from_query.clone().or_else(|| cookie(&req, COOKIE)); if supplied.is_some_and(|t| constant_time_eq(&t, &token)) { - user = admin_user(&state); + user = admin_user(&state).await; if from_query.is_some() { // The token link is a sign-in; the cookie it leaves behind is not one each time. if let Some(u) = &user { - let _ = state.ctx.db.signed_in(u.id); + let _ = state.ctx.db.signed_in(u.id).await; } set_cookie = Some(format!( "{COOKIE}={token}; Path=/; HttpOnly; SameSite=Lax; Max-Age=31536000" @@ -247,8 +247,8 @@ fn cookie(req: &Request, name: &str) -> Option { } /// The account the shared token stands for: the first admin, or the first user at all. -fn admin_user(state: &WebState) -> Option { - let users = state.ctx.db.users().ok()?; +async fn admin_user(state: &WebState) -> Option { + let users = state.ctx.db.users().await.ok()?; users .iter() .find(|u| u.is_admin) @@ -267,7 +267,7 @@ async fn login( Json(body): Json, ) -> Result { let name = body.name.trim().to_ascii_lowercase(); - let user = state.ctx.db.user_by_name(&name)?; + let user = state.ctx.db.user_by_name(&name).await?; // The same answer either way: whether a name exists is not something to leak. let ok = user .as_ref() @@ -280,8 +280,8 @@ async fn login( let user = user.expect("verified above"); let token = crate::auth::new_session_token(); - state.ctx.db.create_session(user.id, &token)?; - state.ctx.db.signed_in(user.id)?; + state.ctx.db.create_session(user.id, &token).await?; + state.ctx.db.signed_in(user.id).await?; tracing::info!(user = %user.name, "signed in"); let days = state.ctx.cfg().web.session_days.max(1); @@ -298,7 +298,7 @@ async fn login( async fn logout(State(state): State, req: Request) -> Response { if let Some(sid) = cookie(&req, SESSION_COOKIE) { - let _ = state.ctx.db.delete_session(&sid); + let _ = state.ctx.db.delete_session(&sid).await; } let mut resp = StatusCode::NO_CONTENT.into_response(); for c in [ @@ -320,7 +320,7 @@ async fn me( ) -> Json { let url = state.ctx.cfg().web.sign_out_url.clone(); let sign_out = (by_proxy && !url.is_empty()).then_some(url); - let (theme, mode) = state.ctx.db.theme(user.id).unwrap_or_default(); + let (theme, mode) = state.ctx.db.theme(user.id).await.unwrap_or_default(); Json(serde_json::json!({ "name": user.name, "admin": user.is_admin, "sign_out": sign_out, "theme": theme, "mode": mode, })) @@ -343,7 +343,7 @@ async fn patch_me( if !theme_ok(&body.theme, &body.mode) { return Err(ApiError::bad_request("not a theme")); } - state.ctx.db.set_theme(user.id, &body.theme, &body.mode)?; + state.ctx.db.set_theme(user.id, &body.theme, &body.mode).await?; Ok(StatusCode::NO_CONTENT) } @@ -378,7 +378,7 @@ async fn list_users( let users: Vec<_> = state .ctx .db - .users()? + .users().await? .iter() .map(|u| { serde_json::json!({ @@ -409,7 +409,7 @@ async fn add_user( let name = crate::auth::name_from_header(&body.name).ok_or_else(|| { ApiError::bad_request("a name is required, without commas, semicolons or line breaks") })?; - if state.ctx.db.user_by_name(&name)?.is_some() { + if state.ctx.db.user_by_name(&name).await?.is_some() { return Err(ApiError::bad_request(format!("{name} already exists"))); } // No password is someone the proxy signs in, as with `ipx user add --no-password`. @@ -418,7 +418,7 @@ async fn add_user( } else { Some(crate::auth::hash_password(&body.password).map_err(|e| ApiError::bad_request(format!("{e:#}")))?) }; - state.ctx.db.create_user(&name, hash.as_deref(), body.admin)?; + state.ctx.db.create_user(&name, hash.as_deref(), body.admin).await?; tracing::info!(by = %user.name, user = %name, admin = body.admin, "account added"); Ok(StatusCode::CREATED) } @@ -435,7 +435,7 @@ async fn patch_user( Json(body): Json, ) -> Result { require_admin(&user)?; - let users = state.ctx.db.users()?; + let users = state.ctx.db.users().await?; let target = users .iter() .find(|u| u.id == id) @@ -446,7 +446,7 @@ async fn patch_user( target.name ))); } - state.ctx.db.set_admin(id, body.admin)?; + state.ctx.db.set_admin(id, body.admin).await?; tracing::info!(by = %user.name, user = %target.name, admin = body.admin, "admin changed"); Ok(StatusCode::NO_CONTENT) } @@ -457,7 +457,7 @@ async fn remove_user( Path(id): Path, ) -> Result { require_admin(&user)?; - let users = state.ctx.db.users()?; + let users = state.ctx.db.users().await?; let target = users .iter() .find(|u| u.id == id) @@ -468,7 +468,7 @@ async fn remove_user( target.name ))); } - state.ctx.db.delete_user(id)?; + state.ctx.db.delete_user(id).await?; tracing::info!(by = %user.name, user = %target.name, "account removed"); Ok(StatusCode::NO_CONTENT) } @@ -510,7 +510,7 @@ async fn admin_page(State(state): State, user: crate::db::User) -> Res if !user.is_admin { return Redirect::to("/").into_response(); } - let theme = state.ctx.db.theme(user.id).unwrap_or_default(); + let theme = state.ctx.db.theme(user.id).await.unwrap_or_default(); let page = with_theme(include_str!(concat!(env!("OUT_DIR"), "/admin.html")), theme); ([(header::CACHE_CONTROL, PAGE_CACHE)], Html(page)).into_response() } @@ -584,7 +584,7 @@ const ADMIN_LINK: &str = ", user: crate::db::User) -> impl IntoResponse { - let theme = state.ctx.db.theme(user.id).unwrap_or_default(); + let theme = state.ctx.db.theme(user.id).await.unwrap_or_default(); ([(header::CACHE_CONTROL, PAGE_CACHE)], Html(page_for(user.is_admin, theme))) } From d7f8f2df1d4a8d42704e33bbaf867098a9654a17 Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 18 Sep 2026 18:23:37 +0000 Subject: [PATCH 3/9] SeaORM: subscriptions and pins Twelve subscription functions move to SeaORM. Lookups use the entity API; the joins, counts and upserts are SQL written to run on both databases: $n parameters, ON CONFLICT DO NOTHING in place of INSERT OR IGNORE, and CASE WHEN on the yes/no column itself rather than comparing it to 1, which Postgres would refuse for a boolean. INSERT ... SELECT ... ON CONFLICT gets a WHERE true, which SQLite needs to tell the two apart. Checked with a daemon on a copy of production: the feed list, read through the new code, comes back with every feed and its settings. Co-Authored-By: Claude Opus 5 --- src/db.rs | 347 +++++++++++++++++++++++++++------------------------- src/main.rs | 20 +-- src/web.rs | 58 ++++----- 3 files changed, 219 insertions(+), 206 deletions(-) diff --git a/src/db.rs b/src/db.rs index 9fbb9bc..a4034ee 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,10 +1,13 @@ //! SQLite state. Replaces the per-feed .ipxd plists, history.dat and qmcache.dat. use anyhow::{Context, Result}; -use crate::entity::{sessions, users}; +use crate::entity::{sessions, subscriptions, users}; use rusqlite::{Connection, OptionalExtension, params}; use sea_orm::sea_query::{Expr, Func}; -use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, QueryOrder, Set}; +use sea_orm::{ + ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, Set, + Statement, +}; use std::path::Path; use std::sync::Mutex; @@ -203,6 +206,23 @@ pub struct User { pub last_login: Option, } +/// A subscription's keywords, stored as a JSON array. +fn keywords(json: Option) -> Option> { + json.and_then(|j| serde_json::from_str(&j).ok()) +} + +impl From for Sub { + fn from(s: subscriptions::Model) -> Self { + Sub { + feed_id: s.feed_id, + keywords: keywords(s.keywords), + auto_download: s.auto_download, + allow_explicit: s.allow_explicit, + max_new_per_check: s.max_new_per_check, + } + } +} + impl From for User { fn from(u: users::Model) -> Self { User { @@ -986,212 +1006,205 @@ impl Db { Ok(()) } + // ---- hand-written SQL ---- + // + // For what reads better as SQL than as a query builder: joins, sums, upserts. Written to + // run on SQLite and Postgres alike -- $1-style parameters, ON CONFLICT, CASE WHEN on a + // yes/no column rather than comparing it to 1 -- since both run it (issue #18). + + fn stmt(&self, sql: &str, values: Vec) -> Statement { + Statement::from_sql_and_values(self.orm.get_database_backend(), sql, values) + } + + /// Runs a statement; how many rows it changed. + async fn exec(&self, sql: &str, values: Vec) -> Result { + Ok(self.orm.execute_raw(self.stmt(sql, values)).await?.rows_affected()) + } + + async fn rows(&self, sql: &str, values: Vec) -> Result> { + Ok(self.orm.query_all_raw(self.stmt(sql, values)).await?) + } + /// The first admin starts subscribed to the whole catalogue: whoever wrote config.toml meant /// to read those feeds, and without this a fresh install signs in to an empty sidebar. Runs /// only while nobody subscribes to anything, so an unsubscribe is never undone. - pub fn adopt_catalogue(&self, user_id: i64, catalogue: &[String]) -> Result { - let conn = self.conn.lock().unwrap(); - let already: i64 = - conn.query_row("SELECT count(*) FROM subscriptions", [], |r| r.get(0))?; - if already > 0 { + pub async fn adopt_catalogue(&self, user_id: i64, catalogue: &[String]) -> Result { + if subscriptions::Entity::find().count(&self.orm).await? > 0 { return Ok(0); } for id in catalogue { - conn.execute( - "INSERT OR IGNORE INTO subscriptions (user_id, feed_id) VALUES (?1, ?2)", - params![user_id, id], - )?; + self.subscribe(user_id, id).await?; } - // Feeds that exist only in the database (OPML children) count too. - conn.execute( - "INSERT OR IGNORE INTO subscriptions (user_id, feed_id) SELECT ?1, id FROM feeds", - params![user_id], - )?; + // Feeds that exist only in the database (OPML children) count too. SQLite wants the + // WHERE to tell the SELECT from the ON CONFLICT; Postgres does not mind it. + self.exec( + "INSERT INTO subscriptions (user_id, feed_id) SELECT $1, id FROM feeds WHERE true + ON CONFLICT DO NOTHING", + vec![user_id.into()], + ) + .await?; Ok(catalogue.len()) } // ---- subscriptions ---- /// What this person wants from a feed. Absent means they do not subscribe at all. - pub fn subscription(&self, user_id: i64, feed_id: &str) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT keywords, auto_download, allow_explicit, max_new_per_check - FROM subscriptions WHERE user_id = ?1 AND feed_id = ?2", - )?; - let mut rows = stmt.query(params![user_id, feed_id])?; - Ok(match rows.next()? { - Some(r) => Some(Sub { - feed_id: feed_id.to_string(), - keywords: r - .get::<_, Option>(0)? - .and_then(|j| serde_json::from_str(&j).ok()), - auto_download: r.get::<_, Option>(1)?.map(|v| v != 0), - allow_explicit: r.get::<_, Option>(2)?.map(|v| v != 0), - max_new_per_check: r.get(3)?, - }), - None => None, - }) + pub async fn subscription(&self, user_id: i64, feed_id: &str) -> Result> { + Ok(subscriptions::Entity::find_by_id((user_id, feed_id.to_owned())) + .one(&self.orm) + .await? + .map(Sub::from)) } /// Every feed this person subscribes to, with their settings. - pub fn subscriptions_for(&self, user_id: i64) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT feed_id, keywords, auto_download, allow_explicit, max_new_per_check - FROM subscriptions WHERE user_id = ?1", - )?; - let out = stmt - .query_map([user_id], |r| { - Ok(Sub { - feed_id: r.get(0)?, - keywords: r - .get::<_, Option>(1)? - .and_then(|j| serde_json::from_str(&j).ok()), - auto_download: r.get::<_, Option>(2)?.map(|v| v != 0), - allow_explicit: r.get::<_, Option>(3)?.map(|v| v != 0), - max_new_per_check: r.get(4)?, - }) - })? - .collect::>>()?; - Ok(out) + pub async fn subscriptions_for(&self, user_id: i64) -> Result> { + Ok(subscriptions::Entity::find() + .filter(subscriptions::Column::UserId.eq(user_id)) + .all(&self.orm) + .await? + .into_iter() + .map(Sub::from) + .collect()) } /// Everyone's settings for one feed. The scanner merges these into what it fetches /// and downloads, since one file serves the lot. /// In a group, whatever someone has not set on the feed itself comes from their /// subscription to the group, as the group's settings dialog has always said it does. - pub fn subscribers(&self, feed_id: &str, group: Option<&str>) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT coalesce(c.keywords, p.keywords), coalesce(c.auto_download, p.auto_download), - coalesce(c.allow_explicit, p.allow_explicit), - coalesce(c.max_new_per_check, p.max_new_per_check) - FROM subscriptions c - LEFT JOIN subscriptions p ON p.user_id = c.user_id AND p.feed_id = ?2 - WHERE c.feed_id = ?1", - )?; - let out = stmt - .query_map(params![feed_id, group], |r| { + pub async fn subscribers(&self, feed_id: &str, group: Option<&str>) -> Result> { + let rows = self + .rows( + "SELECT coalesce(c.keywords, p.keywords) AS keywords, + coalesce(c.auto_download, p.auto_download) AS auto_download, + coalesce(c.allow_explicit, p.allow_explicit) AS allow_explicit, + coalesce(c.max_new_per_check, p.max_new_per_check) AS max_new_per_check + FROM subscriptions c + LEFT JOIN subscriptions p ON p.user_id = c.user_id AND p.feed_id = $2 + WHERE c.feed_id = $1", + vec![feed_id.into(), group.map(str::to_owned).into()], + ) + .await?; + rows.iter() + .map(|r| { Ok(Sub { - feed_id: feed_id.to_string(), - keywords: r - .get::<_, Option>(0)? - .and_then(|j| serde_json::from_str(&j).ok()), - auto_download: r.get::<_, Option>(1)?.map(|v| v != 0), - allow_explicit: r.get::<_, Option>(2)?.map(|v| v != 0), - max_new_per_check: r.get(3)?, + feed_id: feed_id.to_owned(), + keywords: keywords(r.try_get("", "keywords")?), + auto_download: r.try_get("", "auto_download")?, + allow_explicit: r.try_get("", "allow_explicit")?, + max_new_per_check: r.try_get("", "max_new_per_check")?, }) - })? - .collect::>>()?; - Ok(out) + }) + .collect() } /// Subscribers per feed, for the whole catalogue in one query -- the feed list would /// otherwise ask once per feed. - pub fn subscriber_counts(&self) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = - conn.prepare("SELECT feed_id, count(*) FROM subscriptions GROUP BY feed_id")?; - let out = stmt - .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))? - .collect::>>()?; - Ok(out) + pub async fn subscriber_counts(&self) -> Result> { + self.rows("SELECT feed_id, count(*) AS n FROM subscriptions GROUP BY feed_id", vec![]) + .await? + .iter() + .map(|r| Ok((r.try_get("", "feed_id")?, r.try_get("", "n")?))) + .collect() } /// Feeds with any audio or video enclosure: the Directory's Podcasts, with the rest Blogs. /// Reaped files keep their rows, so a show whose files have all been purged still counts. - pub fn media_feeds(&self) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( + pub async fn media_feeds(&self) -> Result> { + self.rows( "SELECT DISTINCT feed_id FROM enclosures WHERE mime LIKE 'audio/%' OR mime LIKE 'video/%'", - )?; - let out = stmt.query_map([], |r| r.get(0))?.collect::>()?; - Ok(out) + vec![], + ) + .await? + .iter() + .map(|r| Ok(r.try_get("", "feed_id")?)) + .collect() } /// Who else would miss this file: subscribers other than `user_id` who have starred /// the item or have not read it yet. Deleting is deleting their copy too. - pub fn others_wanting(&self, enclosure_id: i64, user_id: i64) -> Result<(i64, i64)> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT - sum(CASE WHEN coalesce(st.flagged, 0) = 1 THEN 1 ELSE 0 END), - sum(CASE WHEN coalesce(st.read, 0) = 0 THEN 1 ELSE 0 END) - FROM enclosures e - JOIN subscriptions s ON s.feed_id = e.feed_id AND s.user_id != ?2 - LEFT JOIN entry_state st - ON st.user_id = s.user_id AND st.feed_id = e.feed_id AND st.guid = e.guid - WHERE e.id = ?1", - )?; - let (starred, unread) = stmt.query_row(params![enclosure_id, user_id], |r| { - Ok((r.get::<_, Option>(0)?.unwrap_or(0), r.get::<_, Option>(1)?.unwrap_or(0))) - })?; - Ok((starred, unread)) + pub async fn others_wanting(&self, enclosure_id: i64, user_id: i64) -> Result<(i64, i64)> { + // CASE WHEN on the column itself, not `= 1`: a boolean on Postgres, 0 or 1 on SQLite, + // and NULL, for someone who never opened the item, falls to the ELSE either way. + let r = self + .rows( + "SELECT sum(CASE WHEN st.flagged THEN 1 ELSE 0 END) AS starred, + sum(CASE WHEN st.read THEN 0 ELSE 1 END) AS unread + FROM enclosures e + JOIN subscriptions s ON s.feed_id = e.feed_id AND s.user_id <> $2 + LEFT JOIN entry_state st + ON st.user_id = s.user_id AND st.feed_id = e.feed_id AND st.guid = e.guid + WHERE e.id = $1", + vec![enclosure_id.into(), user_id.into()], + ) + .await?; + let r = r.first().context("an aggregate always returns a row")?; + Ok(( + r.try_get::>("", "starred")?.unwrap_or(0), + r.try_get::>("", "unread")?.unwrap_or(0), + )) } - pub fn subscribe(&self, user_id: i64, feed_id: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "INSERT OR IGNORE INTO subscriptions (user_id, feed_id) VALUES (?1, ?2)", - params![user_id, feed_id], - )?; + pub async fn subscribe(&self, user_id: i64, feed_id: &str) -> Result<()> { + self.exec( + "INSERT INTO subscriptions (user_id, feed_id) VALUES ($1, $2) ON CONFLICT DO NOTHING", + vec![user_id.into(), feed_id.into()], + ) + .await?; Ok(()) } /// The feeds this person pinned to the top of their list. Kept apart from `Sub`, which is /// what the scanner merges into its policy, and which a pin has nothing to do with. - pub fn pinned_feeds(&self, user_id: i64) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare("SELECT feed_id FROM subscriptions WHERE user_id = ?1 AND pinned")?; - let ids = stmt.query_map([user_id], |r| r.get(0))?.collect::>()?; - Ok(ids) + pub async fn pinned_feeds(&self, user_id: i64) -> Result> { + Ok(subscriptions::Entity::find() + .filter(subscriptions::Column::UserId.eq(user_id)) + .filter(subscriptions::Column::Pinned.eq(true)) + .all(&self.orm) + .await? + .into_iter() + .map(|s| s.feed_id) + .collect()) } /// False when they do not subscribe to it, since there is then no row in their list to pin. - pub fn set_pinned(&self, user_id: i64, feed_id: &str, on: bool) -> Result { - let conn = self.conn.lock().unwrap(); - Ok(conn.execute( - "UPDATE subscriptions SET pinned = ?3 WHERE user_id = ?1 AND feed_id = ?2", - params![user_id, feed_id, on as i64], - )? > 0) + pub async fn set_pinned(&self, user_id: i64, feed_id: &str, on: bool) -> Result { + let r = subscriptions::Entity::update_many() + .col_expr(subscriptions::Column::Pinned, Expr::val(on).into()) + .filter(subscriptions::Column::UserId.eq(user_id)) + .filter(subscriptions::Column::FeedId.eq(feed_id)) + .exec(&self.orm) + .await?; + Ok(r.rows_affected > 0) } - pub fn unsubscribe(&self, user_id: i64, feed_id: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "DELETE FROM subscriptions WHERE user_id = ?1 AND feed_id = ?2", - params![user_id, feed_id], - )?; + pub async fn unsubscribe(&self, user_id: i64, feed_id: &str) -> Result<()> { + subscriptions::Entity::delete_by_id((user_id, feed_id.to_owned())).exec(&self.orm).await?; Ok(()) } /// Overwrites one person's settings for a feed. A None field means: follow the feed. - pub fn set_subscription(&self, user_id: i64, sub: &Sub) -> Result<()> { - let conn = self.conn.lock().unwrap(); - let kw = sub - .keywords - .as_ref() - .map(|k| serde_json::to_string(k)) - .transpose()?; - conn.execute( + /// Names its columns, so the pin, which is not a setting, is left as it was. + pub async fn set_subscription(&self, user_id: i64, sub: &Sub) -> Result<()> { + let kw = sub.keywords.as_ref().map(serde_json::to_string).transpose()?; + self.exec( "INSERT INTO subscriptions (user_id, feed_id, keywords, auto_download, allow_explicit, max_new_per_check) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(user_id, feed_id) DO UPDATE SET + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (user_id, feed_id) DO UPDATE SET keywords = excluded.keywords, auto_download = excluded.auto_download, allow_explicit = excluded.allow_explicit, max_new_per_check = excluded.max_new_per_check", - params![ - user_id, - sub.feed_id, - kw, - sub.auto_download.map(|v| v as i64), - sub.allow_explicit.map(|v| v as i64), - sub.max_new_per_check, + vec![ + user_id.into(), + sub.feed_id.clone().into(), + kw.into(), + sub.auto_download.into(), + sub.allow_explicit.into(), + sub.max_new_per_check.into(), ], - )?; + ) + .await?; Ok(()) } @@ -1796,17 +1809,17 @@ mod tests { .unwrap(); // Nobody has touched it: both others still have it unplayed. - assert_eq!(db.others_wanting(1, 1).unwrap(), (0, 2)); + assert_eq!(db.others_wanting(1, 1).await.unwrap(), (0, 2)); // Sam reads it, Kit stars it. db.set_entry_flag(2, "f", "a", EntryFlag::Read, true).unwrap(); db.set_entry_flag(3, "f", "a", EntryFlag::Flagged, true).unwrap(); - assert_eq!(db.others_wanting(1, 1).unwrap(), (1, 1), "one starred it, one has not played it"); + assert_eq!(db.others_wanting(1, 1).await.unwrap(), (1, 1), "one starred it, one has not played it"); // Asking as Kit, only Ray and Sam count -- and Kit's own star is not a reason to // warn Kit. db.set_entry_flag(1, "f", "a", EntryFlag::Read, true).unwrap(); - assert_eq!(db.others_wanting(1, 3).unwrap(), (0, 0)); + assert_eq!(db.others_wanting(1, 3).await.unwrap(), (0, 0)); } #[tokio::test] @@ -1917,12 +1930,12 @@ mod tests { let db = Db::memory().await.unwrap(); let ray = db.create_user("rays", None, true).await.unwrap(); db.create_user("sam", None, false).await.unwrap(); - db.subscribe(ray, "f").unwrap(); + db.subscribe(ray, "f").await.unwrap(); db.rename_user(ray, "rays@sdf1.net").await.unwrap(); assert!(db.user_by_name("rays").await.unwrap().is_none()); let renamed = db.user_by_name("RAYS@sdf1.net").await.unwrap().unwrap(); assert_eq!((renamed.id, renamed.is_admin), (ray, true), "same account, still the admin"); - assert_eq!(db.subscriptions_for(ray).unwrap().len(), 1, "and still subscribed"); + assert_eq!(db.subscriptions_for(ray).await.unwrap().len(), 1, "and still subscribed"); assert!(db.rename_user(ray, "sam").await.is_err(), "a taken name is refused"); } @@ -1955,11 +1968,11 @@ mod tests { db.conn.lock().unwrap().query_row("SELECT count(*) FROM subscriptions", [], |r| r.get(0)).unwrap() }; let catalogue = ["a".to_string(), "b".to_string()]; - assert_eq!(db.adopt_catalogue(1, &catalogue).unwrap(), 2); + assert_eq!(db.adopt_catalogue(1, &catalogue).await.unwrap(), 2); assert_eq!(subs(), 2); // Once anyone subscribes to anything it never runs again, so an unsubscribe sticks. - db.unsubscribe(1, "a").unwrap(); - assert_eq!(db.adopt_catalogue(1, &catalogue).unwrap(), 0); + db.unsubscribe(1, "a").await.unwrap(); + assert_eq!(db.adopt_catalogue(1, &catalogue).await.unwrap(), 0); assert_eq!(subs(), 1); } @@ -2121,14 +2134,14 @@ mod tests { (1,'group',1),(1,'show',NULL),(2,'group',1),(2,'show',0);", ) .unwrap(); - let explicit = |group| -> Vec> { + let explicit = async |group| -> Vec> { let mut v: Vec<_> = - db.subscribers("show", group).unwrap().into_iter().map(|s| s.allow_explicit).collect(); + db.subscribers("show", group).await.unwrap().into_iter().map(|s| s.allow_explicit).collect(); v.sort(); v }; - assert_eq!(explicit(Some("group")), [Some(false), Some(true)], "ray inherits; sam's own choice on the show wins"); - assert_eq!(explicit(None), [None, Some(false)], "outside a group nothing is inherited"); + assert_eq!(explicit(Some("group")).await, [Some(false), Some(true)], "ray inherits; sam's own choice on the show wins"); + assert_eq!(explicit(None).await, [None, Some(false)], "outside a group nothing is inherited"); } #[tokio::test] @@ -2155,15 +2168,15 @@ mod tests { async fn a_pin_survives_saving_the_feeds_settings_and_needs_a_subscription() { let db = Db::memory().await.unwrap(); let me = db.create_user("pat", None, false).await.unwrap(); - assert!(!db.set_pinned(me, "f", true).unwrap(), "not subscribed: nothing to pin"); - db.subscribe(me, "f").unwrap(); - assert!(db.set_pinned(me, "f", true).unwrap()); + assert!(!db.set_pinned(me, "f", true).await.unwrap(), "not subscribed: nothing to pin"); + db.subscribe(me, "f").await.unwrap(); + assert!(db.set_pinned(me, "f", true).await.unwrap()); // set_subscription writes the rest of the row; it must leave the pin alone. - db.set_subscription(me, &Sub { feed_id: "f".into(), auto_download: Some(false), ..Default::default() }) + db.set_subscription(me, &Sub { feed_id: "f".into(), auto_download: Some(false), ..Default::default() }).await .unwrap(); - assert!(db.pinned_feeds(me).unwrap().contains("f")); - db.set_pinned(me, "f", false).unwrap(); - assert!(db.pinned_feeds(me).unwrap().is_empty()); + assert!(db.pinned_feeds(me).await.unwrap().contains("f")); + db.set_pinned(me, "f", false).await.unwrap(); + assert!(db.pinned_feeds(me).await.unwrap().is_empty()); } #[tokio::test] diff --git a/src/main.rs b/src/main.rs index 764a4a5..a2ccf42 100644 --- a/src/main.rs +++ b/src/main.rs @@ -380,7 +380,7 @@ async fn daemon( if let Some(admin) = ctx.db.users().await?.into_iter().find(|u| u.is_admin) { let catalogue: Vec = ctx.cfg().feeds.keys().cloned().collect(); - match ctx.db.adopt_catalogue(admin.id, &catalogue) { + match ctx.db.adopt_catalogue(admin.id, &catalogue).await { Ok(0) => {} Ok(n) => tracing::info!(user = %admin.name, feeds = n, "subscribed the first admin to the catalogue"), Err(e) => tracing::error!(error = %e, "could not subscribe the first admin to the catalogue"), @@ -676,7 +676,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path .ok_or_else(|| anyhow::anyhow!("no admin account to subscribe: ipx user add --admin"))?; let doc = opml::OPML::from_str(&text) .map_err(|e| anyhow::anyhow!("{} is not OPML: {e}", file.display()))?; - let (added, had) = subscribe_opml(ctx, config_path, &doc, admin.id)?; + let (added, had) = subscribe_opml(ctx, config_path, &doc, admin.id).await?; println!("subscribed {} to {added} feed(s); {had} already there", admin.name); Ok(()) } @@ -691,7 +691,7 @@ async fn import(ctx: &Ctx, config_path: &std::path::Path, file: &std::path::Path /// /// The caller parses the document, so each refuses a file that is not OPML in its own terms, /// before anything is touched: a 400 from the web, a message from the CLI. -pub fn subscribe_opml( +pub async fn subscribe_opml( ctx: &Ctx, config_path: &std::path::Path, doc: &opml::OPML, @@ -746,10 +746,10 @@ pub fn subscribe_opml( let (mut added, mut had) = (0, 0); for id in ids { - if ctx.db.subscription(user_id, &id)?.is_some() { + if ctx.db.subscription(user_id, &id).await?.is_some() { had += 1; } else { - ctx.db.subscribe(user_id, &id)?; + ctx.db.subscribe(user_id, &id).await?; added += 1; } } @@ -1136,7 +1136,7 @@ async fn scan_one( parsed.category.as_deref(), )?; - let policy = policy_for(ctx, id, feed_cfg)?; + let policy = policy_for(ctx, id, feed_cfg).await?; if let Some(parent) = &feed_cfg.group { let listed: Vec<(&str, &str)> = parsed .entries @@ -1318,8 +1318,8 @@ async fn sync_group( .chain(std::iter::once(parent_id.to_string())) { for user in ctx.db.users().await? { - if ctx.db.subscription(user.id, parent_id)?.is_some() { - ctx.db.subscribe(user.id, &id)?; + if ctx.db.subscription(user.id, parent_id).await?.is_some() { + ctx.db.subscribe(user.id, &id).await?; } } } @@ -1404,9 +1404,9 @@ pub struct Policy { pub budget: usize, } -fn policy_for(ctx: &Ctx, id: &str, feed_cfg: &config::Feed) -> Result { +async fn policy_for(ctx: &Ctx, id: &str, feed_cfg: &config::Feed) -> Result { let global = ctx.cfg().general.max_new_per_check; - Ok(merge_policy(&ctx.db.subscribers(id, feed_cfg.group.as_deref())?, feed_cfg, global)) + Ok(merge_policy(&ctx.db.subscribers(id, feed_cfg.group.as_deref()).await?, feed_cfg, global)) } fn merge_policy(subs: &[db::Sub], feed_cfg: &config::Feed, global: usize) -> Policy { diff --git a/src/web.rs b/src/web.rs index 5c4fd72..2f05130 100644 --- a/src/web.rs +++ b/src/web.rs @@ -676,12 +676,12 @@ async fn feeds( let mine: std::collections::HashMap = state .ctx .db - .subscriptions_for(user.id)? + .subscriptions_for(user.id).await? .into_iter() .map(|s| (s.feed_id.clone(), s)) .collect(); - let counts = state.ctx.db.subscriber_counts()?; - let pinned = state.ctx.db.pinned_feeds(user.id)?; + let counts = state.ctx.db.subscriber_counts().await?; + let pinned = state.ctx.db.pinned_feeds(user.id).await?; let mut out = Vec::with_capacity(mine.len()); for sub in &subs { let (id, feed) = (&sub.id, &sub.cfg); @@ -800,12 +800,12 @@ struct PopularRow { /// first. Popular is the top of it, the directory is all of it, and it is all that /// `subscribe_popular` will subscribe you to. An OPML or a Patreon creator is listed as the /// feeds inside it and never itself: both lists are for finding a show. -fn popular(state: &WebState, user_id: i64) -> Result> { +async fn popular(state: &WebState, user_id: i64) -> Result> { let db = &state.ctx.db; let mine: std::collections::HashSet = - db.subscriptions_for(user_id)?.into_iter().map(|s| s.feed_id).collect(); - let counts = db.subscriber_counts()?; - let media = db.media_feeds()?; + db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect(); + let counts = db.subscriber_counts().await?; + let media = db.media_feeds().await?; let catalogue = crate::subscriptions(&state.ctx)?; let by_id: std::collections::HashMap<&str, &crate::config::Feed> = catalogue.iter().map(|s| (s.id.as_str(), &s.cfg)).collect(); @@ -844,7 +844,7 @@ async fn get_popular( State(state): State, user: crate::db::User, ) -> Result>, ApiError> { - let mut rows = popular(&state, user.id)?; + let mut rows = popular(&state, user.id).await?; rows.truncate(10); Ok(Json(rows)) } @@ -854,7 +854,7 @@ async fn get_directory( State(state): State, user: crate::db::User, ) -> Result>, ApiError> { - let mut rows = popular(&state, user.id)?; + let mut rows = popular(&state, user.id).await?; rows.sort_by_key(sort_name); Ok(Json(rows)) } @@ -870,10 +870,10 @@ async fn subscribe_popular( user: crate::db::User, Path(id): Path, ) -> Result, ApiError> { - if !popular(&state, user.id)?.iter().any(|p| p.id == id) { + if !popular(&state, user.id).await?.iter().any(|p| p.id == id) { return Err(ApiError::bad_request(format!("{id:?} is not in the directory"))); } - state.ctx.db.subscribe(user.id, &id)?; + state.ctx.db.subscribe(user.id, &id).await?; Ok(Json(serde_json::json!({ "id": id }))) } @@ -1202,10 +1202,10 @@ struct NewFeed { /// The Add feed dialog's explicit box. Like everything on a feed's own dialog it is yours, so it /// goes on your subscription, and before the first scan, which would otherwise skip every /// explicit item. -fn explicit_on_add(state: &WebState, user_id: i64, feed_id: &str, allow: bool) -> Result<(), ApiError> { +async fn explicit_on_add(state: &WebState, user_id: i64, feed_id: &str, allow: bool) -> Result<(), ApiError> { if allow { let sub = crate::db::Sub { feed_id: feed_id.to_owned(), allow_explicit: Some(true), ..Default::default() }; - state.ctx.db.set_subscription(user_id, &sub)?; + state.ctx.db.set_subscription(user_id, &sub).await?; } Ok(()) } @@ -1223,10 +1223,10 @@ async fn add_feed( .into_iter() .find(|s| crate::feed::same_feed(&s.cfg.url, &url)) { - let already = state.ctx.db.subscription(user.id, &existing.id)?.is_some(); - state.ctx.db.subscribe(user.id, &existing.id)?; + let already = state.ctx.db.subscription(user.id, &existing.id).await?.is_some(); + state.ctx.db.subscribe(user.id, &existing.id).await?; if !already { - explicit_on_add(&state, user.id, &existing.id, body.allow_explicit)?; + explicit_on_add(&state, user.id, &existing.id, body.allow_explicit).await?; } scan_soon(&state, Some(existing.id.clone())).await; return Ok(Json( @@ -1236,8 +1236,8 @@ async fn add_feed( let id = crate::add_one(&state.ctx, &mut cfg, &url, body.folder, body.keywords).await?; cfg.save(&state.config_path)?; state.ctx.reload_cfg(&state.config_path)?; - state.ctx.db.subscribe(user.id, &id)?; - explicit_on_add(&state, user.id, &id, body.allow_explicit)?; + state.ctx.db.subscribe(user.id, &id).await?; + explicit_on_add(&state, user.id, &id, body.allow_explicit).await?; scan_soon(&state, Some(id.clone())).await; Ok(Json(serde_json::json!({ "id": id, "existing": false }))) } @@ -1290,17 +1290,17 @@ async fn patch_feed( ) -> Result { // Pinning is yours alone too, and means nothing for a feed you do not subscribe to. if let Some(on) = body.pinned - && !state.ctx.db.set_pinned(user.id, &id, on)? + && !state.ctx.db.set_pinned(user.id, &id, on).await? { return Err(ApiError::not_found("you do not subscribe to that feed")); } // What one person wants -- which items, whether to fetch them, how many at a time -- // is theirs. It goes on their subscription and nobody else sees the change. - if state.ctx.db.subscription(user.id, &id)?.is_some() { + if state.ctx.db.subscription(user.id, &id).await?.is_some() { let mut mine = state .ctx .db - .subscription(user.id, &id)? + .subscription(user.id, &id).await? .unwrap_or_else(|| crate::db::Sub { feed_id: id.clone(), ..Default::default() }); let mut touched = false; if let Some(v) = body.keywords.clone() { @@ -1320,7 +1320,7 @@ async fn patch_feed( touched = true; } if touched { - state.ctx.db.set_subscription(user.id, &mine)?; + state.ctx.db.set_subscription(user.id, &mine).await?; } } @@ -1400,14 +1400,14 @@ async fn remove_feed( ) -> Result { // Unsubscribing is personal: it takes the feed off your list and leaves everyone // else's alone. - state.ctx.db.unsubscribe(user.id, &id)?; + state.ctx.db.unsubscribe(user.id, &id).await?; for child in crate::subscriptions(&state.ctx)? .iter() .filter(|s| s.cfg.group.as_deref() == Some(id.as_str())) { - state.ctx.db.unsubscribe(user.id, &child.id)?; + state.ctx.db.unsubscribe(user.id, &child.id).await?; } - if state.ctx.db.subscriber_counts()?.contains_key(&id) { + if state.ctx.db.subscriber_counts().await?.contains_key(&id) { return Ok(StatusCode::NO_CONTENT); } @@ -1493,7 +1493,7 @@ async fn delete_file( // There is one copy of the file: deleting it deletes everyone's. Say so before doing // it, once, and let them decide. if !q.force { - let (starred, unread) = state.ctx.db.others_wanting(id, user.id)?; + let (starred, unread) = state.ctx.db.others_wanting(id, user.id).await?; let people = |n: i64| if n == 1 { "person".to_string() } else { format!("{n} people") }; let complaint = match (starred, unread) { (0, 0) => None, @@ -1626,7 +1626,7 @@ async fn read_all_mine( user: crate::db::User, ) -> Result, ApiError> { let ids: Vec = - state.ctx.db.subscriptions_for(user.id)?.into_iter().map(|s| s.feed_id).collect(); + state.ctx.db.subscriptions_for(user.id).await?.into_iter().map(|s| s.feed_id).collect(); let n = state.ctx.db.mark_all_read(user.id, &ids)?; Ok(Json(serde_json::json!({ "marked": n }))) } @@ -1668,7 +1668,7 @@ async fn export_opml( // Yours, not the whole catalogue: other people's feeds, and any private URLs in them, are // not yours to download. This used to export config.toml to whoever asked. let mine: std::collections::HashSet = - state.ctx.db.subscriptions_for(user.id)?.into_iter().map(|s| s.feed_id).collect(); + state.ctx.db.subscriptions_for(user.id).await?.into_iter().map(|s| s.feed_id).collect(); let mut doc = opml::OPML { head: Some(opml::Head { title: Some("ipx subscriptions".into()), @@ -1718,7 +1718,7 @@ async fn import_opml( // file arrives as text, is read here, and is gone when the request ends. let doc = opml::OPML::from_str(&body.xml) .map_err(|e| ApiError::bad_request(format!("that is not an OPML file: {e}")))?; - let (added, already) = crate::subscribe_opml(&state.ctx, &state.config_path, &doc, user.id)?; + let (added, already) = crate::subscribe_opml(&state.ctx, &state.config_path, &doc, user.id).await?; if added > 0 { scan_soon(&state, None).await; } From 6bf1ad6b31cc161bfb7e103298af3298e8022c28 Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 18 Sep 2026 18:32:46 +0000 Subject: [PATCH 4/9] SeaORM: items and read state The item list, its counts, filters, sorts and search, positions, pins and mark-all-read move to SeaORM, as SQL written for both databases: - Parameters are gathered as the SQL is written (Args), so only what a statement uses is bound. rusqlite needed every one mentioned, hence the old `?1 IS NULL` and `?2 = ''`; Postgres refuses a parameter it cannot type. - Yes/no columns are tested as booleans (NOT coalesce(s.read, false)) and written as true, not 1; SQLite reads true and false as 1 and 0. - The last tiebreak of the sort is the guid, not SQLite's rowid, which Postgres lacks. Only items with the same date change places. - set_position names entry_state.duration beside excluded.duration. - The status callback on the control socket returns a future, as the counts are now a query. Checked on a copy of production against the live server: 42 of 48 lists identical; the other six differ only in how ties fall, or because the test daemon cleared paths to files this machine does not have. Run on the same file, every filter's count matches the old SQL exactly. Co-Authored-By: Claude Opus 5 --- src/db.rs | 399 +++++++++++++++++++++++++++------------------------- src/ipc.rs | 9 +- src/main.rs | 11 +- src/web.rs | 22 +-- 4 files changed, 234 insertions(+), 207 deletions(-) diff --git a/src/db.rs b/src/db.rs index a4034ee..93b3beb 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,7 +1,7 @@ //! SQLite state. Replaces the per-feed .ipxd plists, history.dat and qmcache.dat. use anyhow::{Context, Result}; -use crate::entity::{sessions, subscriptions, users}; +use crate::entity::{enclosures, sessions, subscriptions, users}; use rusqlite::{Connection, OptionalExtension, params}; use sea_orm::sea_query::{Expr, Func}; use sea_orm::{ @@ -206,6 +206,22 @@ pub struct User { pub last_login: Option, } +impl From for EncRow { + fn from(e: enclosures::Model) -> Self { + EncRow { + id: e.id, + feed_id: e.feed_id, + guid: e.guid, + url: e.url, + mime: e.mime, + length: e.length, + path: e.path, + state: e.state, + last_error: e.last_error, + } + } +} + /// A subscription's keywords, stored as a JSON array. fn keywords(json: Option) -> Option> { json.and_then(|j| serde_json::from_str(&j).ok()) @@ -722,25 +738,30 @@ impl Db { } impl Db { - pub fn unread_count(&self, user_id: i64, feed_id: &str) -> Result { - let conn = self.conn.lock().unwrap(); - Ok(conn.query_row( - "SELECT count(*) FROM entries e - LEFT JOIN entry_state s - ON s.user_id = ?2 AND s.feed_id = e.feed_id AND s.guid = e.guid - WHERE e.feed_id = ?1 AND coalesce(s.read, 0) = 0", - rusqlite::params![feed_id, user_id], - |r| r.get(0), - )?) + pub async fn unread_count(&self, user_id: i64, feed_id: &str) -> Result { + let r = self + .rows( + "SELECT count(*) AS n FROM entries e + LEFT JOIN entry_state s + ON s.user_id = $2 AND s.feed_id = e.feed_id AND s.guid = e.guid + WHERE e.feed_id = $1 AND NOT coalesce(s.read, false)", + vec![feed_id.into(), user_id.into()], + ) + .await?; + Ok(r.first().context("count(*) always returns a row")?.try_get("", "n")?) } /// (pending, downloaded) across all feeds, for the status command. - pub fn counts(&self) -> Result<(i64, i64)> { - let conn = self.conn.lock().unwrap(); - Ok(( - conn.query_row("SELECT count(*) FROM enclosures WHERE state = 'pending'", [], |r| r.get(0))?, - conn.query_row("SELECT count(*) FROM enclosures WHERE path IS NOT NULL", [], |r| r.get(0))?, - )) + pub async fn counts(&self) -> Result<(i64, i64)> { + let pending = enclosures::Entity::find() + .filter(enclosures::Column::State.eq("pending")) + .count(&self.orm) + .await?; + let downloaded = enclosures::Entity::find() + .filter(enclosures::Column::Path.is_not_null()) + .count(&self.orm) + .await?; + Ok((pending as i64, downloaded as i64)) } } @@ -764,22 +785,45 @@ pub struct EntryRow { pub enclosures: Vec, } -/// The search clause. `?2` is referenced unconditionally -- binding a parameter the -/// statement does not mention is an error, so an empty needle short-circuits instead. -const SEARCH: &str = "(?2 = '' OR lower(coalesce(e.title, '')) LIKE ?2 - OR lower(coalesce(e.description, '')) LIKE ?2)"; +/// A statement's parameters, gathered as its SQL is written: each `p` binds a value and gives +/// back its `$n`. Only what the SQL uses is bound -- Postgres refuses a parameter it cannot +/// place, where rusqlite needed every one mentioned, which is what `?1 IS NULL` was for. +#[derive(Default)] +struct Args(Vec); -/// Which feeds a query covers: one, or every feed the person subscribes to. Both forms -/// mention `?1`, since binding a parameter the statement does not use is an error. -fn scope_sql(feed_id: Option<&str>, user_param: u8) -> String { - match feed_id { - Some(_) => "e.feed_id = ?1".into(), - None => format!( - "?1 IS NULL AND e.feed_id IN (SELECT feed_id FROM subscriptions WHERE user_id = ?{user_param})" - ), +impl Args { + fn p(&mut self, v: impl Into) -> String { + self.0.push(v.into()); + format!("${}", self.0.len()) } } +/// The FROM and WHERE that `entries_in` and `count_in` share: whose read state, which feeds +/// (one, or every feed the person subscribes to), the filter, and a case-insensitive search of +/// title and description. +fn entries_from(a: &mut Args, user_id: i64, feed_id: Option<&str>, filter: Filter, search: Option<&str>) -> String { + let user = a.p(user_id); + let scope = match feed_id { + Some(f) => format!("e.feed_id = {}", a.p(f)), + None => format!("e.feed_id IN (SELECT feed_id FROM subscriptions WHERE user_id = {user})"), + }; + let search = match search.map(|q| q.trim().to_lowercase()).filter(|q| !q.is_empty()) { + Some(q) => { + let like = a.p(format!("%{q}%")); + format!( + "(lower(coalesce(e.title, '')) LIKE {like} OR lower(coalesce(e.description, '')) LIKE {like})" + ) + } + None => "true".into(), + }; + format!( + "FROM entries e + LEFT JOIN entry_state s ON s.user_id = {user} AND s.feed_id = e.feed_id AND s.guid = e.guid + WHERE {scope} AND {} AND {search}", + filter.sql() + ) +} + /// The item table's ORDER BY. The column name picks one of these fixed expressions, so nothing /// the caller sends reaches the query, and anything unrecognised is newest first. Ties fall back /// to newest first too, so a page boundary is stable across "Load more". @@ -791,7 +835,7 @@ fn scope_sql(feed_id: Option<&str>, user_param: u8) -> String { /// where the direction chosen is the point. pub fn order_sql(col: &str, dir: &str, pinned_first: bool) -> String { let expr = match col { - "kept" => "coalesce(s.flagged, 0)", + "kept" => "coalesce(s.flagged, false)", "title" => "lower(coalesce(e.title, ''))", "feed" => "(SELECT lower(coalesce(f.title, f.id)) FROM feeds f WHERE f.id = e.feed_id)", "type" => "(SELECT min(x.mime) FROM enclosures x WHERE x.feed_id = e.feed_id AND x.guid = e.guid)", @@ -799,8 +843,9 @@ pub fn order_sql(col: &str, dir: &str, pinned_first: bool) -> String { _ => "coalesce(e.published, e.first_seen)", }; let dir = if dir == "asc" { "ASC" } else { "DESC" }; - let pins = if pinned_first && col != "kept" { "coalesce(s.flagged, 0) DESC, " } else { "" }; - format!("{pins}{expr} {dir}, coalesce(e.published, e.first_seen) DESC, e.rowid DESC") + let pins = if pinned_first && col != "kept" { "coalesce(s.flagged, false) DESC, " } else { "" }; + // The guid breaks what ties remain: SQLite's rowid did, and Postgres has none. + format!("{pins}{expr} {dir}, coalesce(e.published, e.first_seen) DESC, e.guid DESC") } /// Which slice of a feed the UI is asking for. @@ -834,9 +879,11 @@ impl Filter { /// correlated against it. fn sql(self) -> &'static str { match self { - Self::All => "1=1", - Self::Unread => "coalesce(s.read, 0) = 0", - Self::Flagged => "coalesce(s.flagged, 0) = 1", + // true and false, not 1 and 0: the columns are booleans on Postgres, and SQLite + // reads true and false as 1 and 0. + Self::All => "true", + Self::Unread => "NOT coalesce(s.read, false)", + Self::Flagged => "coalesce(s.flagged, false)", Self::Downloaded => { "EXISTS (SELECT 1 FROM enclosures x WHERE x.feed_id = e.feed_id AND x.guid = e.guid AND x.path IS NOT NULL)" @@ -867,7 +914,8 @@ impl Db { /// One page of entries, each with its enclosures attached: one feed's, or every feed the /// person subscribes to when `feed_id` is None (All Subscriptions). `search` matches title /// and description, case-insensitively. - pub fn entries_in( + #[allow(clippy::too_many_arguments)] + pub async fn entries_in( &self, user_id: i64, feed_id: Option<&str>, @@ -877,108 +925,73 @@ impl Db { limit: i64, order: &str, ) -> Result> { - let conn = self.conn.lock().unwrap(); - let like = search - .map(|q| format!("%{}%", q.trim().to_lowercase())) - .unwrap_or_default(); + let mut a = Args::default(); + let from = entries_from(&mut a, user_id, feed_id, filter, search); let sql = format!( "SELECT e.guid, e.feed_id, e.title, e.link, e.published, e.description, - coalesce(s.read, 0), coalesce(s.flagged, 0), e.image, - coalesce(s.duration, e.duration), - e.episode, e.season, coalesce(s.position, 0) - FROM entries e - LEFT JOIN entry_state s - ON s.user_id = ?5 AND s.feed_id = e.feed_id AND s.guid = e.guid - WHERE {} AND {} AND {SEARCH} + coalesce(s.read, false) AS read, coalesce(s.flagged, false) AS flagged, e.image, + coalesce(s.duration, e.duration) AS duration, + e.episode, e.season, coalesce(s.position, 0) AS position + {from} ORDER BY {order} - LIMIT ?4 OFFSET ?3", - scope_sql(feed_id, 5), - filter.sql() + LIMIT {} OFFSET {}", + a.p(limit), + a.p(offset) ); - let mut stmt = conn.prepare(&sql)?; - let map = |r: &rusqlite::Row| -> rusqlite::Result { - Ok(EntryRow { - guid: r.get(0)?, - feed_id: r.get(1)?, - title: r.get(2)?, - link: r.get(3)?, - published: r.get(4)?, - description: r.get(5)?, - read: r.get::<_, i64>(6)? != 0, - flagged: r.get::<_, i64>(7)? != 0, - image: r.get(8)?, - duration: r.get(9)?, - episode: r.get(10)?, - season: r.get(11)?, - position: r.get(12)?, - enclosures: vec![], + let mut rows = self + .rows(&sql, a.0) + .await? + .iter() + .map(|r| { + Ok(EntryRow { + guid: r.try_get("", "guid")?, + feed_id: r.try_get("", "feed_id")?, + title: r.try_get("", "title")?, + link: r.try_get("", "link")?, + published: r.try_get("", "published")?, + description: r.try_get("", "description")?, + read: r.try_get("", "read")?, + flagged: r.try_get("", "flagged")?, + image: r.try_get("", "image")?, + duration: r.try_get("", "duration")?, + episode: r.try_get("", "episode")?, + season: r.try_get("", "season")?, + position: r.try_get("", "position")?, + enclosures: vec![], + }) }) - }; - let mut rows: Vec = stmt - .query_map(rusqlite::params![feed_id, like, offset, limit, user_id], map)? - .collect::>>()?; - + .collect::>>()?; if rows.is_empty() { return Ok(rows); } // Only the guids on this page, so a feed with thousands of entries stays cheap. A page // can span feeds, so each file is matched to its row by feed as well as guid, below. - let placeholders = std::iter::repeat_n("?", rows.len()).collect::>().join(","); - let sql = format!( - "SELECT id, feed_id, guid, url, mime, length, path, state, last_error - FROM enclosures WHERE guid IN ({placeholders}) ORDER BY id" - ); - let mut params: Vec<&dyn rusqlite::ToSql> = Vec::with_capacity(rows.len()); - for row in &rows { - params.push(&row.guid); - } - let mut stmt = conn.prepare(&sql)?; - let encs = stmt - .query_map(params.as_slice(), |r| { - Ok(EncRow { - id: r.get(0)?, - feed_id: r.get(1)?, - guid: r.get(2)?, - url: r.get(3)?, - mime: r.get(4)?, - length: r.get(5)?, - path: r.get(6)?, - state: r.get(7)?, - last_error: r.get(8)?, - }) - })? - .collect::>>()?; - + let encs = enclosures::Entity::find() + .filter(enclosures::Column::Guid.is_in(rows.iter().map(|r| r.guid.clone()))) + .order_by_asc(enclosures::Column::Id) + .all(&self.orm) + .await?; for enc in encs { if let Some(row) = rows.iter_mut().find(|r| r.guid == enc.guid && r.feed_id == enc.feed_id) { - row.enclosures.push(enc); + row.enclosures.push(EncRow::from(enc)); } } Ok(rows) } /// How many entries `entries_in` would page through, so the UI knows whether there is more. - pub fn count_in( + pub async fn count_in( &self, user_id: i64, feed_id: Option<&str>, filter: Filter, search: Option<&str>, ) -> Result { - let conn = self.conn.lock().unwrap(); - let like = search - .map(|q| format!("%{}%", q.trim().to_lowercase())) - .unwrap_or_default(); - let sql = format!( - "SELECT count(*) FROM entries e - LEFT JOIN entry_state s - ON s.user_id = ?3 AND s.feed_id = e.feed_id AND s.guid = e.guid - WHERE {} AND {} AND {SEARCH}", - scope_sql(feed_id, 3), - filter.sql() - ); - Ok(conn.query_row(&sql, rusqlite::params![feed_id, like, user_id], |r| r.get(0))?) + let mut a = Args::default(); + let sql = format!("SELECT count(*) AS n {}", entries_from(&mut a, user_id, feed_id, filter, search)); + let r = self.rows(&sql, a.0).await?; + Ok(r.first().context("count(*) always returns a row")?.try_get("", "n")?) } /// Where playback got to, so it resumes there next time -- for this listener only. @@ -987,7 +1000,7 @@ impl Db { /// out: ReThinking's gave 41:23 for a 43:48 file, which said 0:08 left with 2:33 to play. /// Not written to `entries`, where every scan puts the feed's figure back, and kept per /// listener so one person's player never changes what anyone else sees. - pub fn set_position( + pub async fn set_position( &self, user_id: i64, feed_id: &str, @@ -995,14 +1008,21 @@ impl Db { secs: i64, duration: Option, ) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( + // entry_state.duration, named in full: Postgres will not guess between it and excluded's. + self.exec( "INSERT INTO entry_state (user_id, feed_id, guid, position, duration) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(user_id, feed_id, guid) DO UPDATE SET position = excluded.position, - duration = coalesce(excluded.duration, duration)", - rusqlite::params![user_id, feed_id, guid, secs.max(0), duration.filter(|d| *d > 0)], - )?; + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (user_id, feed_id, guid) DO UPDATE SET position = excluded.position, + duration = coalesce(excluded.duration, entry_state.duration)", + vec![ + user_id.into(), + feed_id.into(), + guid.into(), + secs.max(0).into(), + duration.filter(|d| *d > 0).into(), + ], + ) + .await?; Ok(()) } @@ -1361,19 +1381,20 @@ impl Db { /// Marks every entry of the given feeds read. Takes a list because an OPML subscription /// holds no entries itself -- marking it read means the feeds inside it. - pub fn mark_all_read(&self, user_id: i64, feed_ids: &[String]) -> Result { - let conn = self.conn.lock().unwrap(); + pub async fn mark_all_read(&self, user_id: i64, feed_ids: &[String]) -> Result { let mut n = 0; for id in feed_ids { - n += conn.execute( - "INSERT INTO entry_state (user_id, feed_id, guid, read) - SELECT ?1, e.feed_id, e.guid, 1 FROM entries e - LEFT JOIN entry_state s - ON s.user_id = ?1 AND s.feed_id = e.feed_id AND s.guid = e.guid - WHERE e.feed_id = ?2 AND coalesce(s.read, 0) = 0 - ON CONFLICT(user_id, feed_id, guid) DO UPDATE SET read = 1", - rusqlite::params![user_id, id], - )?; + n += self + .exec( + "INSERT INTO entry_state (user_id, feed_id, guid, read) + SELECT $1, e.feed_id, e.guid, true FROM entries e + LEFT JOIN entry_state s + ON s.user_id = $1 AND s.feed_id = e.feed_id AND s.guid = e.guid + WHERE e.feed_id = $2 AND NOT coalesce(s.read, false) + ON CONFLICT (user_id, feed_id, guid) DO UPDATE SET read = true", + vec![user_id.into(), id.clone().into()], + ) + .await? as usize; } Ok(n) } @@ -1419,7 +1440,7 @@ impl Db { } /// Read and kept, per person. The row is created on first touch. - pub fn set_entry_flag( + pub async fn set_entry_flag( &self, user_id: i64, feed_id: &str, @@ -1427,19 +1448,19 @@ impl Db { field: EntryFlag, on: bool, ) -> Result<()> { - let conn = self.conn.lock().unwrap(); let col = match field { EntryFlag::Read => "read", EntryFlag::Flagged => "flagged", }; - conn.execute( + self.exec( &format!( "INSERT INTO entry_state (user_id, feed_id, guid, {col}) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(user_id, feed_id, guid) DO UPDATE SET {col} = excluded.{col}" + VALUES ($1, $2, $3, $4) + ON CONFLICT (user_id, feed_id, guid) DO UPDATE SET {col} = excluded.{col}" ), - rusqlite::params![user_id, feed_id, guid, on as i64], - )?; + vec![user_id.into(), feed_id.into(), guid.into(), on.into()], + ) + .await?; Ok(()) } @@ -1765,34 +1786,34 @@ mod tests { (3,'f','c','u3','video/mp4',2000,'pending');", ) .unwrap(); - let order = |col: &str, dir: &str| -> Vec { - db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir, false)) + let order = async |col: &str, dir: &str| -> Vec { + db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir, false)).await .unwrap() .into_iter() .map(|e| e.guid) .collect() }; - assert_eq!(order("title", "asc"), ["b", "a", "c"], "Apple, banana, cherry: case folded"); - assert_eq!(order("title", "desc"), ["c", "a", "b"]); - assert_eq!(order("feed", "asc"), ["b", "c", "a"], "Aardvark, then Zebra's newest first"); - assert_eq!(order("type", "asc"), ["a", "b", "c"], "audio, image, video"); - assert_eq!(order("size", "desc"), ["c", "a", "b"]); - assert_eq!(order("published", "desc"), ["c", "b", "a"]); - db.set_entry_flag(1, "f", "a", EntryFlag::Flagged, true).unwrap(); - assert_eq!(order("kept", "desc")[0], "a"); + assert_eq!(order("title", "asc").await, ["b", "a", "c"], "Apple, banana, cherry: case folded"); + assert_eq!(order("title", "desc").await, ["c", "a", "b"]); + assert_eq!(order("feed", "asc").await, ["b", "c", "a"], "Aardvark, then Zebra's newest first"); + assert_eq!(order("type", "asc").await, ["a", "b", "c"], "audio, image, video"); + assert_eq!(order("size", "desc").await, ["c", "a", "b"]); + assert_eq!(order("published", "desc").await, ["c", "b", "a"]); + db.set_entry_flag(1, "f", "a", EntryFlag::Flagged, true).await.unwrap(); + assert_eq!(order("kept", "desc").await[0], "a"); // Pinned first: the pinned banana tops every sort, the rest in the order asked for. - let pinned = |col: &str, dir: &str| -> Vec { - db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir, true)) + let pinned = async |col: &str, dir: &str| -> Vec { + db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir, true)).await .unwrap() .into_iter() .map(|e| e.guid) .collect() }; - assert_eq!(pinned("published", "desc"), ["a", "c", "b"]); - assert_eq!(pinned("title", "desc"), ["a", "c", "b"]); - assert_eq!(pinned("kept", "asc")[2], "a", "sorting by the pin itself keeps its direction"); + assert_eq!(pinned("published", "desc").await, ["a", "c", "b"]); + assert_eq!(pinned("title", "desc").await, ["a", "c", "b"]); + assert_eq!(pinned("kept", "asc").await[2], "a", "sorting by the pin itself keeps its direction"); // An unknown column or direction is newest first; the name itself never reaches the SQL. - assert_eq!(order("title; DROP TABLE entries", "sideways"), ["c", "b", "a"]); + assert_eq!(order("title; DROP TABLE entries", "sideways").await, ["c", "b", "a"]); assert!(!order_sql("x'; --", "asc", false).contains("x'")); } @@ -1812,13 +1833,13 @@ mod tests { assert_eq!(db.others_wanting(1, 1).await.unwrap(), (0, 2)); // Sam reads it, Kit stars it. - db.set_entry_flag(2, "f", "a", EntryFlag::Read, true).unwrap(); - db.set_entry_flag(3, "f", "a", EntryFlag::Flagged, true).unwrap(); + db.set_entry_flag(2, "f", "a", EntryFlag::Read, true).await.unwrap(); + db.set_entry_flag(3, "f", "a", EntryFlag::Flagged, true).await.unwrap(); assert_eq!(db.others_wanting(1, 1).await.unwrap(), (1, 1), "one starred it, one has not played it"); // Asking as Kit, only Ray and Sam count -- and Kit's own star is not a reason to // warn Kit. - db.set_entry_flag(1, "f", "a", EntryFlag::Read, true).unwrap(); + db.set_entry_flag(1, "f", "a", EntryFlag::Read, true).await.unwrap(); assert_eq!(db.others_wanting(1, 3).await.unwrap(), (0, 0)); } @@ -1832,19 +1853,19 @@ mod tests { ) .unwrap(); - assert_eq!(db.unread_count(1, "f").unwrap(), 2); - assert_eq!(db.unread_count(2, "f").unwrap(), 2); + assert_eq!(db.unread_count(1, "f").await.unwrap(), 2); + assert_eq!(db.unread_count(2, "f").await.unwrap(), 2); - db.set_entry_flag(1, "f", "a", EntryFlag::Read, true).unwrap(); - assert_eq!(db.unread_count(1, "f").unwrap(), 1, "ray read one of them"); - assert_eq!(db.unread_count(2, "f").unwrap(), 2, "sam has read nothing"); + db.set_entry_flag(1, "f", "a", EntryFlag::Read, true).await.unwrap(); + assert_eq!(db.unread_count(1, "f").await.unwrap(), 1, "ray read one of them"); + assert_eq!(db.unread_count(2, "f").await.unwrap(), 2, "sam has read nothing"); // Starring and position are just as private. - db.set_entry_flag(1, "f", "b", EntryFlag::Flagged, true).unwrap(); - db.set_position(2, "f", "b", 42, Some(600)).unwrap(); + db.set_entry_flag(1, "f", "b", EntryFlag::Flagged, true).await.unwrap(); + db.set_position(2, "f", "b", 42, Some(600)).await.unwrap(); let order = order_sql("published", "desc", false); - let page = |user| db.entries_in(user, Some("f"), Filter::All, None, 0, 50, &order).unwrap(); - let (ray, sam) = (page(1), page(2)); + let page = async |user| db.entries_in(user, Some("f"), Filter::All, None, 0, 50, &order).await.unwrap(); + let (ray, sam) = (page(1).await, page(2).await); let ray_b = ray.iter().find(|e| e.guid == "b").unwrap(); let sam_b = sam.iter().find(|e| e.guid == "b").unwrap(); assert!(ray_b.flagged && ray_b.position == 0); @@ -1854,9 +1875,9 @@ mod tests { assert_eq!(sam_b.duration, Some(600)); // Marking a whole feed read is likewise one person's business. - assert_eq!(db.mark_all_read(2, &["f".to_string()]).unwrap(), 2); - assert_eq!(db.unread_count(2, "f").unwrap(), 0); - assert_eq!(db.unread_count(1, "f").unwrap(), 1); + assert_eq!(db.mark_all_read(2, &["f".to_string()]).await.unwrap(), 2); + assert_eq!(db.unread_count(2, "f").await.unwrap(), 0); + assert_eq!(db.unread_count(1, "f").await.unwrap(), 1); } #[tokio::test] @@ -2013,42 +2034,42 @@ mod tests { for f in [Filter::All, Filter::Unread, Filter::Downloaded, Filter::Flagged, Filter::InProgress] { // Both paths must run without erroring, and agree with each other. let order = order_sql("published", "desc", false); - let rows = db.entries_in(7, Some("f"), f, None, 0, 50, &order).unwrap(); - let n = db.count_in(7, Some("f"), f, None).unwrap(); + let rows = db.entries_in(7, Some("f"), f, None, 0, 50, &order).await.unwrap(); + let n = db.count_in(7, Some("f"), f, None).await.unwrap(); assert_eq!(rows.len() as i64, n, "{f:?} count disagrees with the page"); - let rows = db.entries_in(7, Some("f"), f, Some("dive"), 0, 50, &order).unwrap(); - let n = db.count_in(7, Some("f"), f, Some("dive")).unwrap(); + let rows = db.entries_in(7, Some("f"), f, Some("dive"), 0, 50, &order).await.unwrap(); + let n = db.count_in(7, Some("f"), f, Some("dive")).await.unwrap(); assert_eq!(rows.len() as i64, n, "{f:?} with search disagrees"); } - assert_eq!(db.count_in(7, Some("f"), Filter::All, None).unwrap(), 7); - assert_eq!(db.count_in(7, Some("f"), Filter::Unread, None).unwrap(), 3); - assert_eq!(db.count_in(7, Some("f"), Filter::Downloaded, None).unwrap(), 1); - assert_eq!(db.count_in(7, Some("f"), Filter::Flagged, None).unwrap(), 1); - assert_eq!(db.count_in(7, Some("f"), Filter::All, Some("dive")).unwrap(), 2); - assert_eq!(db.count_in(7, Some("f"), Filter::All, Some("NOTES two")).unwrap(), 1, + assert_eq!(db.count_in(7, Some("f"), Filter::All, None).await.unwrap(), 7); + assert_eq!(db.count_in(7, Some("f"), Filter::Unread, None).await.unwrap(), 3); + assert_eq!(db.count_in(7, Some("f"), Filter::Downloaded, None).await.unwrap(), 1); + assert_eq!(db.count_in(7, Some("f"), Filter::Flagged, None).await.unwrap(), 1); + assert_eq!(db.count_in(7, Some("f"), Filter::All, Some("dive")).await.unwrap(), 2); + assert_eq!(db.count_in(7, Some("f"), Filter::All, Some("NOTES two")).await.unwrap(), 1, "search is case-insensitive and covers the description"); // Currently Listening: started, not finished, and not just an accidental tap. let order = order_sql("published", "desc", false); - let listening = || { - let rows = db.entries_in(7, Some("f"), Filter::InProgress, None, 0, 50, &order).unwrap(); + let listening = async || { + let rows = db.entries_in(7, Some("f"), Filter::InProgress, None, 0, 50, &order).await.unwrap(); rows.into_iter().map(|e| e.guid).collect::>() }; - assert_eq!(listening(), ["h", "d"]); + assert_eq!(listening().await, ["h", "d"]); // The player's measured length is the one that counts, in place of a missing one or over // the feed's: d, 42 of a measured 45 seconds, is finished; e, which its feed calls 45 // seconds long, is 42 into a 9000-second file and is not. Times left use it too. - db.set_position(7, "f", "d", 42, Some(45)).unwrap(); - db.set_position(7, "f", "e", 42, Some(9000)).unwrap(); - assert_eq!(listening(), ["h", "e"]); - let rows = db.entries_in(7, Some("f"), Filter::All, None, 0, 50, &order).unwrap(); + db.set_position(7, "f", "d", 42, Some(45)).await.unwrap(); + db.set_position(7, "f", "e", 42, Some(9000)).await.unwrap(); + assert_eq!(listening().await, ["h", "e"]); + let rows = db.entries_in(7, Some("f"), Filter::All, None, 0, 50, &order).await.unwrap(); assert_eq!(rows.iter().find(|r| r.guid == "e").unwrap().duration, Some(9000)); // A save without one (before the player knows) keeps the length already measured. - db.set_position(7, "f", "e", 43, None).unwrap(); - assert_eq!(listening(), ["h", "e"]); + db.set_position(7, "f", "e", 43, None).await.unwrap(); + assert_eq!(listening().await, ["h", "e"]); } #[tokio::test] diff --git a/src/ipc.rs b/src/ipc.rs index 453f404..231cdd1 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -176,7 +176,9 @@ pub async fn daemon_is_live(path: &Path) -> bool { /// healthcheck left waiting behind a scan or a long download timed out and called a busy daemon /// dead. The answer goes to the client that asked and no one else: broadcast, it ended any /// `ipx fetch` that was watching a scan, since `status` is a terminal event. -pub type StatusFn = std::sync::Arc Event + Send + Sync>; +/// A future, since reading the counts is a database query. +pub type StatusFn = + std::sync::Arc std::pin::Pin + Send>> + Send + Sync>; /// Accepts connections, feeding commands to `cmds` and events from `events` back out. pub async fn serve( @@ -249,7 +251,7 @@ async fn handle( // Answered here, not queued behind whatever the worker is on: see StatusFn. Ok(Command::Status) => { tracing::info!(target: "ipx::io", "-> {line}"); - let ev = status(); + let ev = status().await; if let Ok(json) = serde_json::to_string(&ev) { tracing::info!(target: "ipx::io", "<- {json}"); } @@ -366,7 +368,8 @@ mod tests { // Another client, watching a scan: it must not be handed someone else's answer, which // would end its session. let mut watcher = events.subscribe(); - let status: StatusFn = std::sync::Arc::new(|| Event::Status { feeds: 1, pending: 2, downloaded: 3 }); + let status: StatusFn = + std::sync::Arc::new(|| Box::pin(async { Event::Status { feeds: 1, pending: 2, downloaded: 3 } })); let (client, server) = UnixStream::pair().unwrap(); tokio::spawn(handle(server, events.subscribe(), cmds, status)); diff --git a/src/main.rs b/src/main.rs index a2ccf42..56321fb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -340,7 +340,7 @@ async fn run(ctx: &Arc, cmd: Cmd) -> Result<()> { Cmd::Reap { dry_run } => reap(ctx, dry_run, true), Cmd::Download { enclosure } => download_one(ctx, enclosure).await, Cmd::Status => { - ctx.out.emit(status(ctx)); + ctx.out.emit(status(ctx).await); Ok(()) } } @@ -348,8 +348,8 @@ async fn run(ctx: &Arc, cmd: Cmd) -> Result<()> { /// The counts `ipx status` prints. A running daemon's socket answers with this directly rather /// than through the job queue. -fn status(ctx: &Ctx) -> Event { - match ctx.db.counts() { +async fn status(ctx: &Ctx) -> Event { + match ctx.db.counts().await { Ok((pending, downloaded)) => { let feeds = subscriptions(ctx).map(|s| s.len()).unwrap_or(0); Event::Status { feeds, pending, downloaded } @@ -420,7 +420,10 @@ async fn daemon( // status is answered by the socket itself; everything else waits its turn in the queue. let answer: ipc::StatusFn = { let ctx = ctx.clone(); - Arc::new(move || status(&ctx)) + Arc::new(move || { + let ctx = ctx.clone(); + Box::pin(async move { status(&ctx).await }) + }) }; let server = tokio::spawn(ipc::serve(socket.clone(), events.clone(), tx_cmd, answer)); diff --git a/src/web.rs b/src/web.rs index 2f05130..67a36e8 100644 --- a/src/web.rs +++ b/src/web.rs @@ -740,7 +740,7 @@ async fn feeds( last_error: s.last_error, entries: s.entries, downloaded: s.downloaded, - unread: state.ctx.db.unread_count(user.id, id)?, + unread: state.ctx.db.unread_count(user.id, id).await?, subscribers: counts.get(id).copied().unwrap_or(0), pinned: pinned.contains(id), }); @@ -1124,7 +1124,7 @@ async fn entries( user: crate::db::User, Query(page): Query, ) -> Result, ApiError> { - entry_page(&state, user.id, Some(&id), &page) + entry_page(&state, user.id, Some(&id), &page).await } /// Every subscribed feed's items together, newest first: All Subscriptions. @@ -1133,11 +1133,11 @@ async fn all_entries( user: crate::db::User, Query(page): Query, ) -> Result, ApiError> { - entry_page(&state, user.id, None, &page) + entry_page(&state, user.id, None, &page).await } /// One feed's page of items, or every subscribed feed's when `feed` is None. -fn entry_page( +async fn entry_page( state: &WebState, user_id: i64, feed: Option<&str>, @@ -1153,14 +1153,14 @@ fn entry_page( filter != crate::db::Filter::InProgress, ); let mut rows = - db.entries_in(user_id, feed, filter, search, page.offset, page.limit.clamp(1, 200), &order)?; + db.entries_in(user_id, feed, filter, search, page.offset, page.limit.clamp(1, 200), &order).await?; let mut sanitizer = feed_sanitizer(); for row in &mut rows { if let Some(d) = &row.description { row.description = Some(clean_description(&mut sanitizer, d, row.link.as_deref())); } } - let total = db.count_in(user_id, feed, filter, search)?; + let total = db.count_in(user_id, feed, filter, search).await?; Ok(Json(EntryPage { total, entries: rows })) } @@ -1440,10 +1440,10 @@ async fn set_flags( ) -> Result { use crate::db::EntryFlag; if let Some(v) = body.read { - state.ctx.db.set_entry_flag(user.id, &feed_id, &guid, EntryFlag::Read, v)?; + state.ctx.db.set_entry_flag(user.id, &feed_id, &guid, EntryFlag::Read, v).await?; } if let Some(v) = body.flagged { - state.ctx.db.set_entry_flag(user.id, &feed_id, &guid, EntryFlag::Flagged, v)?; + state.ctx.db.set_entry_flag(user.id, &feed_id, &guid, EntryFlag::Flagged, v).await?; } Ok(StatusCode::NO_CONTENT) } @@ -1598,7 +1598,7 @@ async fn set_position( user: crate::db::User, Json(body): Json, ) -> Result { - state.ctx.db.set_position(user.id, &feed_id, &guid, body.secs, body.duration)?; + state.ctx.db.set_position(user.id, &feed_id, &guid, body.secs, body.duration).await?; Ok(StatusCode::NO_CONTENT) } @@ -1615,7 +1615,7 @@ async fn read_all( .filter(|s| s.cfg.group.as_deref() == Some(id.as_str())) .map(|s| s.id), ); - let n = state.ctx.db.mark_all_read(user.id, &ids)?; + let n = state.ctx.db.mark_all_read(user.id, &ids).await?; Ok(Json(serde_json::json!({ "marked": n }))) } @@ -1627,7 +1627,7 @@ async fn read_all_mine( ) -> Result, ApiError> { let ids: Vec = state.ctx.db.subscriptions_for(user.id).await?.into_iter().map(|s| s.feed_id).collect(); - let n = state.ctx.db.mark_all_read(user.id, &ids)?; + let n = state.ctx.db.mark_all_read(user.id, &ids).await?; Ok(Json(serde_json::json!({ "marked": n }))) } From a68bfb179b80416d714b5846579c22307967670b Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 18 Sep 2026 18:38:07 +0000 Subject: [PATCH 5/9] SeaORM: enclosures, downloads and the reaper Twelve enclosure functions move to SeaORM: recording, the download queue, marking done or failed, requeueing, and what the reaper may delete. INSERT OR IGNORE becomes ON CONFLICT DO NOTHING; the reaper's read verdict is true or false rather than 1 or 0, which Postgres would type as a 32-bit integer and refuse to read as an i64; `read = 1` and `flagged = 1` test the booleans themselves. retention::run and its callers (reap, rm, retire_group, retire_stranded) become async. The reaper deletes files, so it was checked on a copy of production against the old SQL on the same file: all 2,195 candidates, identical and in the same order. Co-Authored-By: Claude Opus 5 --- src/db.rs | 226 +++++++++++++++++++++-------------------------- src/main.rs | 70 +++++++-------- src/retention.rs | 18 ++-- src/web.rs | 16 ++-- 4 files changed, 154 insertions(+), 176 deletions(-) diff --git a/src/db.rs b/src/db.rs index 93b3beb..dee1bd1 100644 --- a/src/db.rs +++ b/src/db.rs @@ -559,39 +559,46 @@ impl Db { /// Returns true when this enclosure URL is new. False means we have downloaded it /// before, or deliberately reaped it -- either way it is not fetched again. - pub fn mark_downloaded(&self, url: &str, path: &std::path::Path, bytes: u64) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE enclosures SET state = 'done', path = ?2, bytes_done = ?3, - downloaded_at = ?4, last_error = NULL WHERE url = ?1", - rusqlite::params![url, path.to_string_lossy(), bytes as i64, now()], - )?; + pub async fn mark_downloaded(&self, url: &str, path: &std::path::Path, bytes: u64) -> Result<()> { + self.exec( + "UPDATE enclosures SET state = 'done', path = $2, bytes_done = $3, + downloaded_at = $4, last_error = NULL WHERE url = $1", + vec![url.into(), path.to_string_lossy().into_owned().into(), (bytes as i64).into(), now().into()], + ) + .await?; Ok(()) } /// The row stays -- a failed URL is still a URL we have seen. `state` says why it has /// no file, and a retry is an explicit act rather than something a rescan does silently. - pub fn mark_enclosure(&self, url: &str, state: &str, error: Option<&str>) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE enclosures SET state = ?2, last_error = ?3 WHERE url = ?1", - rusqlite::params![url, state, error], - )?; + pub async fn mark_enclosure(&self, url: &str, state: &str, error: Option<&str>) -> Result<()> { + self.exec( + "UPDATE enclosures SET state = $2, last_error = $3 WHERE url = $1", + vec![url.into(), state.into(), error.map(str::to_owned).into()], + ) + .await?; Ok(()) } - pub fn record_enclosure( + pub async fn record_enclosure( &self, feed_id: &str, guid: &str, enc: &crate::feed::Enclosure, ) -> Result { - let conn = self.conn.lock().unwrap(); - let inserted = conn.execute( - "INSERT OR IGNORE INTO enclosures (feed_id, guid, url, mime, length, state) - VALUES (?1, ?2, ?3, ?4, ?5, 'pending')", - rusqlite::params![feed_id, guid, enc.url, enc.mime, enc.length], - )?; + let inserted = self + .exec( + "INSERT INTO enclosures (feed_id, guid, url, mime, length, state) + VALUES ($1, $2, $3, $4, $5, 'pending') ON CONFLICT DO NOTHING", + vec![ + feed_id.into(), + guid.into(), + enc.url.clone().into(), + enc.mime.clone().into(), + enc.length.into(), + ], + ) + .await?; Ok(inserted == 1) } } @@ -608,23 +615,21 @@ pub struct Pending { impl Db { /// The download queue is the table, not the parse result: an enclosure held back by /// `max_new_per_check` is simply picked up by the next scan, in feed order. - pub fn pending(&self, feed_id: &str, limit: usize) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - // Newest first: a cap of 3 should mean the three latest episodes, not the - // three that happen to have been recorded first. + pub async fn pending(&self, feed_id: &str, limit: usize) -> Result> { + // Newest first: a cap of 3 should mean the three latest episodes, not the three that + // happen to have been recorded first. + self.rows( "SELECT x.id, x.url, x.mime FROM enclosures x JOIN entries e ON e.feed_id = x.feed_id AND e.guid = x.guid - WHERE x.feed_id = ?1 AND x.state = 'pending' + WHERE x.feed_id = $1 AND x.state = 'pending' ORDER BY coalesce(e.published, e.first_seen) DESC, x.id DESC - LIMIT ?2", - )?; - let rows = stmt - .query_map(rusqlite::params![feed_id, limit as i64], |r| { - Ok(Pending { id: r.get(0)?, url: r.get(1)?, mime: r.get(2)? }) - })? - .collect::>>()?; - Ok(rows) + LIMIT $2", + vec![feed_id.into(), (limit as i64).into()], + ) + .await? + .iter() + .map(|r| Ok(Pending { id: r.try_get("", "id")?, url: r.try_get("", "url")?, mime: r.try_get("", "mime")? })) + .collect() } } @@ -652,57 +657,53 @@ impl Db { /// /// (The Python intended `read = 1 AND flagged = 0` but never achieved it -- a missing /// plistlib import and an `EntreiesData` typo meant the filter always threw.) - pub fn reap_candidates(&self) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( - "SELECT e.id, e.url, e.path, e.bytes_done, coalesce(e.downloaded_at, 0), - CASE WHEN coalesce(readers.n, 0) >= coalesce(subs.n, 0) THEN 1 ELSE 0 END + pub async fn reap_candidates(&self) -> Result> { + // Yes/no as true and false, not 1 and 0: Postgres types a bare 1 as a 32-bit integer + // and will not hand it over as an i64. + self.rows( + "SELECT e.id, e.url, e.path, e.bytes_done, coalesce(e.downloaded_at, 0) AS age_key, + CASE WHEN coalesce(readers.n, 0) >= coalesce(subs.n, 0) THEN true ELSE false END AS read FROM enclosures e - LEFT JOIN (SELECT feed_id, count(*) n FROM subscriptions GROUP BY feed_id) subs + LEFT JOIN (SELECT feed_id, count(*) AS n FROM subscriptions GROUP BY feed_id) subs ON subs.feed_id = e.feed_id - LEFT JOIN (SELECT feed_id, guid, count(*) n FROM entry_state - WHERE read = 1 GROUP BY feed_id, guid) readers + LEFT JOIN (SELECT feed_id, guid, count(*) AS n FROM entry_state + WHERE read GROUP BY feed_id, guid) readers ON readers.feed_id = e.feed_id AND readers.guid = e.guid WHERE e.path IS NOT NULL AND NOT EXISTS (SELECT 1 FROM entry_state s - WHERE s.feed_id = e.feed_id AND s.guid = e.guid AND s.flagged = 1) + WHERE s.feed_id = e.feed_id AND s.guid = e.guid AND s.flagged) ORDER BY 6 DESC, coalesce(e.downloaded_at, 0) ASC, e.id ASC", - )?; - let rows = stmt - .query_map([], |r| { - Ok(Candidate { - id: r.get(0)?, - url: r.get(1)?, - path: r.get(2)?, - bytes: r.get(3)?, - age_key: r.get(4)?, - read: r.get::<_, i64>(5)? != 0, - }) - })? - .collect::>>()?; - Ok(rows) + vec![], + ) + .await? + .iter() + .map(|r| { + Ok(Candidate { + id: r.try_get("", "id")?, + url: r.try_get("", "url")?, + path: r.try_get("", "path")?, + bytes: r.try_get("", "bytes_done")?, + age_key: r.try_get("", "age_key")?, + read: r.try_get("", "read")?, + }) + }) + .collect() } /// The row survives the file: that is what stops a reaped episode being re-downloaded. - pub fn mark_reaped(&self, id: i64) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE enclosures SET state = 'reaped', path = NULL WHERE id = ?1", - [id], - )?; + pub async fn mark_reaped(&self, id: i64) -> Result<()> { + self.exec("UPDATE enclosures SET state = 'reaped', path = NULL WHERE id = $1", vec![id.into()]).await?; Ok(()) } /// Rows claiming a file that is no longer there (someone deleted it by hand). - pub fn missing_files(&self) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = - conn.prepare("SELECT id, path FROM enclosures WHERE path IS NOT NULL")?; - let rows = stmt - .query_map([], |r| Ok((r.get(0)?, r.get::<_, String>(1)?)))? - .collect::>>()?; - Ok(rows + pub async fn missing_files(&self) -> Result> { + Ok(enclosures::Entity::find() + .filter(enclosures::Column::Path.is_not_null()) + .all(&self.orm) + .await? .into_iter() + .filter_map(|e| Some((e.id, e.path?))) .filter(|(_, p)| !std::path::Path::new(p).exists()) .collect()) } @@ -1401,42 +1402,23 @@ impl Db { /// The next N enclosures with no file, newest entry first -- what "download latest" /// queues up. - pub fn undownloaded(&self, feed_id: &str, limit: i64) -> Result> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( + pub async fn undownloaded(&self, feed_id: &str, limit: i64) -> Result> { + self.rows( "SELECT x.id FROM enclosures x JOIN entries e ON e.feed_id = x.feed_id AND e.guid = x.guid - WHERE x.feed_id = ?1 AND x.path IS NULL AND x.state != 'reaped' + WHERE x.feed_id = $1 AND x.path IS NULL AND x.state <> 'reaped' ORDER BY coalesce(e.published, e.first_seen) DESC - LIMIT ?2", - )?; - Ok(stmt - .query_map(rusqlite::params![feed_id, limit], |r| r.get(0))? - .collect::>>()?) + LIMIT $2", + vec![feed_id.into(), limit.into()], + ) + .await? + .iter() + .map(|r| Ok(r.try_get("", "id")?)) + .collect() } - pub fn enclosure(&self, id: i64) -> Result> { - let conn = self.conn.lock().unwrap(); - Ok(conn - .query_row( - "SELECT id, feed_id, guid, url, mime, length, path, state, last_error - FROM enclosures WHERE id = ?1", - [id], - |r| { - Ok(EncRow { - id: r.get(0)?, - feed_id: r.get(1)?, - guid: r.get(2)?, - url: r.get(3)?, - mime: r.get(4)?, - length: r.get(5)?, - path: r.get(6)?, - state: r.get(7)?, - last_error: r.get(8)?, - }) - }, - ) - .optional()?) + pub async fn enclosure(&self, id: i64) -> Result> { + Ok(enclosures::Entity::find_by_id(id).one(&self.orm).await?.map(EncRow::from)) } /// Read and kept, per person. The row is created on first touch. @@ -1466,13 +1448,12 @@ impl Db { /// How many files this feed has on disk. Decides whether a feed dropped from an OPML /// can be removed or must be kept. - pub fn downloaded_count(&self, feed_id: &str) -> Result { - let conn = self.conn.lock().unwrap(); - Ok(conn.query_row( - "SELECT count(*) FROM enclosures WHERE feed_id = ?1 AND path IS NOT NULL", - [feed_id], - |r| r.get(0), - )?) + pub async fn downloaded_count(&self, feed_id: &str) -> Result { + Ok(enclosures::Entity::find() + .filter(enclosures::Column::FeedId.eq(feed_id)) + .filter(enclosures::Column::Path.is_not_null()) + .count(&self.orm) + .await? as i64) } /// Names a feed without touching its conditional-GET validators. An OPML subscription @@ -1675,23 +1656,20 @@ impl Db { /// Nothing can be in flight the moment the daemon starts, so any row still marked /// `downloading` is a leftover from a restart or a crash. Left alone it would sit /// there forever: the pending queue skips it and nothing else ever revisits it. - pub fn requeue_interrupted(&self) -> Result { - let conn = self.conn.lock().unwrap(); - Ok(conn.execute( - "UPDATE enclosures SET state = 'pending' WHERE state = 'downloading' AND path IS NULL", - [], - )?) + pub async fn requeue_interrupted(&self) -> Result { + Ok(self + .exec("UPDATE enclosures SET state = 'pending' WHERE state = 'downloading' AND path IS NULL", vec![]) + .await? as usize) } /// Puts an enclosure back in the queue so the next scan picks it up. This is how a /// `skipped` verdict (from a filter that has since been changed) gets revisited. - pub fn requeue(&self, id: i64) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE enclosures SET state = 'pending', last_error = NULL - WHERE id = ?1 AND path IS NULL", - [id], - )?; + pub async fn requeue(&self, id: i64) -> Result<()> { + self.exec( + "UPDATE enclosures SET state = 'pending', last_error = NULL WHERE id = $1 AND path IS NULL", + vec![id.into()], + ) + .await?; Ok(()) } } @@ -2085,7 +2063,7 @@ mod tests { (3,'f','new','u-new','pending');", ) .unwrap(); - let got: Vec = db.pending("f", 2).unwrap().into_iter().map(|p| p.url).collect(); + let got: Vec = db.pending("f", 2).await.unwrap().into_iter().map(|p| p.url).collect(); assert_eq!(got, vec!["u-new", "u-mid"], "newest first, oldest left for later"); } @@ -2100,7 +2078,7 @@ mod tests { (4,'f','d','u4','done','/tmp/x');", ) .unwrap(); - assert_eq!(db.requeue_interrupted().unwrap(), 1, "only the in-flight, fileless one"); + assert_eq!(db.requeue_interrupted().await.unwrap(), 1, "only the in-flight, fileless one"); let conn = db.conn.lock().unwrap(); let state = |id: i64| -> String { conn.query_row("SELECT state FROM enclosures WHERE id = ?1", [id], |r| r.get(0)).unwrap() diff --git a/src/main.rs b/src/main.rs index 56321fb..6a227f0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -225,7 +225,7 @@ async fn main() -> Result<()> { Command::Add { url, folder, keywords } => { add(&ctx, &config_path, &url, folder, keywords).await } - Command::Rm { feed } => rm(&ctx, &config_path, &feed), + Command::Rm { feed } => rm(&ctx, &config_path, &feed).await, Command::User { cmd } => user_cmd(&ctx, cmd).await, Command::Import { file } => import(&ctx, &config_path, &file).await, Command::Export { file } => export(&ctx, &file), @@ -334,10 +334,10 @@ async fn run(ctx: &Arc, cmd: Cmd) -> Result<()> { match cmd { Cmd::Fetch { feed, force } => { // Make room before pulling more down, as the original did per download. - reap(ctx, false, false)?; + reap(ctx, false, false).await?; fetch(ctx, feed.as_deref(), force).await } - Cmd::Reap { dry_run } => reap(ctx, dry_run, true), + Cmd::Reap { dry_run } => reap(ctx, dry_run, true).await, Cmd::Download { enclosure } => download_one(ctx, enclosure).await, Cmd::Status => { ctx.out.emit(status(ctx).await); @@ -387,13 +387,13 @@ async fn daemon( } } - match ctx.db.requeue_interrupted() { + match ctx.db.requeue_interrupted().await { Ok(n) if n > 0 => tracing::info!(count = n, "requeued downloads interrupted by a restart"), Ok(_) => {} Err(e) => tracing::warn!(error = ?e, "could not requeue interrupted downloads"), } - match retire_stranded(&ctx) { + match retire_stranded(&ctx).await { Ok(0) => {} Ok(n) => tracing::info!(feeds = n, "retired feeds whose OPML is no longer in config"), Err(e) => tracing::warn!(error = ?e, "could not retire feeds whose OPML is no longer in config"), @@ -651,7 +651,7 @@ fn url_stem(url: &str) -> String { .unwrap_or_else(|| url.to_owned()) } -fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> { +async fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> { let mut cfg = (*ctx.cfg()).clone(); if cfg.feeds.remove(feed).is_none() { // Derived from an OPML: drop it here, though the subscription will list it again @@ -663,7 +663,7 @@ fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> { cfg.save(config_path)?; // State and files stay: re-adding the feed should not re-download its back catalogue. println!("removed {feed}; downloads and history kept"); - retire_group(ctx, feed)?; + retire_group(ctx, feed).await?; Ok(()) } @@ -813,8 +813,8 @@ fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> { /// `standalone` false means this is the sweep that runs before a scan: it reports what it /// deleted, but must not emit the terminal ReapDone, or a client waiting on its `fetch` /// would stop reading before the scan had even started. -fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> { - let r = retention::run(&ctx.cfg(), &ctx.db, dry_run)?; +async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> { + let r = retention::run(&ctx.cfg(), &ctx.db, dry_run).await?; for c in r.aged_out.iter().chain(r.over_quota.iter()) { ctx.out.emit(Event::Reaped { path: c.path.clone(), @@ -992,14 +992,14 @@ pub fn subscriptions(ctx: &Ctx) -> Result> { /// itself is removed, since `subscriptions()` would otherwise keep scanning them under a /// fallback policy meant for a feed with no parent at all. A feed promoted to config is not /// derived any more, so it is only unmanaged. -pub fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> { +pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> { let cfg = ctx.cfg(); for m in ctx.db.managed_feeds()?.into_iter().filter(|m| m.group_id == parent_id) { if cfg.feeds.contains_key(&m.id) { // Scanned from its config entry and still read. Dropped as derived, its stored // entries would go with it: davewiner's 11 were promoted without being unmanaged. ctx.db.unmanage(&m.id)?; - } else if ctx.db.downloaded_count(&m.id).unwrap_or(1) > 0 { + } else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 { ctx.db.set_orphaned(&m.id, true)?; } else { ctx.db.drop_managed(&m.id)?; @@ -1012,7 +1012,7 @@ pub fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> { /// dropped or unmanaged. An OPML removed before `retire_group` existed left its feeds behind: /// davewiner's 922 were skipped by every scan and never cleared, and their stale errors were /// most of the ones stored. -fn retire_stranded(ctx: &Ctx) -> Result { +async fn retire_stranded(ctx: &Ctx) -> Result { let cfg = ctx.cfg(); let before = ctx.db.managed_feeds()?; let stranded: std::collections::BTreeSet<&str> = before @@ -1021,7 +1021,7 @@ fn retire_stranded(ctx: &Ctx) -> Result { .filter(|g| !cfg.feeds.contains_key(*g)) .collect(); for group in stranded { - retire_group(ctx, group)?; + retire_group(ctx, group).await?; } Ok(before.len() - ctx.db.managed_feeds()?.len()) } @@ -1159,7 +1159,7 @@ async fn scan_one( scan.new_entries += 1; } for enc in &entry.enclosures { - let was = if ctx.db.record_enclosure(id, &entry.guid, enc)? { + let was = if ctx.db.record_enclosure(id, &entry.guid, enc).await? { None } else if let Some(reason) = skipped.get(&enc.url) { Some(reason.as_str()) @@ -1169,8 +1169,8 @@ async fn scan_one( let now = reject(&ctx.cfg(), feed_cfg, &policy, entry, enc); if now != was { match now { - Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason))?, - None => ctx.db.mark_enclosure(&enc.url, "pending", None)?, + Some(reason) => ctx.db.mark_enclosure(&enc.url, "skipped", Some(reason)).await?, + None => ctx.db.mark_enclosure(&enc.url, "pending", None).await?, } } } @@ -1182,10 +1182,10 @@ async fn scan_one( let folder = download::folder_for(&cfg, id, feed_cfg, parsed.title.as_deref()); let dest_dir = cfg.general.download_dir.join(&folder); - for item in ctx.db.pending(id, budget)? { + for item in ctx.db.pending(id, budget).await? { if download::looks_like_torrent(&item.url, item.mime.as_deref()) { if !ctx.cfg().torrent.enabled { - ctx.db.mark_enclosure(&item.url, "skipped", Some("torrents disabled"))?; + ctx.db.mark_enclosure(&item.url, "skipped", Some("torrents disabled")).await?; ctx.out.emit(Event::TorrentDeferred { feed: id.to_string(), url: item.url.clone(), @@ -1195,14 +1195,14 @@ async fn scan_one( } if ctx.detach_torrents { // 'downloading' keeps the next scan from queueing it a second time. - ctx.db.mark_enclosure(&item.url, "downloading", None)?; + ctx.db.mark_enclosure(&item.url, "downloading", None).await?; spawn_torrent(ctx, id.to_string(), item.id, item.url.clone(), dest_dir.clone()); scan.torrents += 1; continue; } match torrent_one(ctx, id, item.id, &item.url, &dest_dir).await { Ok((path, bytes)) => { - ctx.db.mark_downloaded(&item.url, &path, bytes)?; + ctx.db.mark_downloaded(&item.url, &path, bytes).await?; ctx.out.emit(Event::DownloadDone { feed: id.to_string(), enclosure: item.id, @@ -1220,7 +1220,7 @@ async fn scan_one( url: item.url.clone(), msg: msg.clone(), }); - ctx.db.mark_enclosure(&item.url, "error", Some(&msg))?; + ctx.db.mark_enclosure(&item.url, "error", Some(&msg)).await?; scan.failed += 1; } } @@ -1245,7 +1245,7 @@ async fn scan_one( url: item.url.clone(), msg: msg.clone(), }); - ctx.db.mark_enclosure(&item.url, "error", Some(&msg))?; + ctx.db.mark_enclosure(&item.url, "error", Some(&msg)).await?; scan.failed += 1; } } @@ -1334,7 +1334,7 @@ async fn sync_group( if listed.iter().any(|(_, u)| u == &m.url) { continue; } - if ctx.db.downloaded_count(&m.id).unwrap_or(1) > 0 { + if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 { // Never orphan a downloaded file: keep the feed and say why in the UI. ctx.db.set_orphaned(&m.id, true)?; kept += 1; @@ -1488,14 +1488,14 @@ async fn fetch_one( // as if it were an episode. let _ = tokio::fs::remove_file(&got.tmp).await; if !ctx.cfg().torrent.enabled { - ctx.db.mark_enclosure(url, "skipped", Some("torrents disabled"))?; + ctx.db.mark_enclosure(url, "skipped", Some("torrents disabled")).await?; anyhow::bail!("body is a torrent and torrents are disabled"); } return torrent_one(ctx, feed_id, enclosure, url, dest_dir).await; } let path = download::place(&got, dest_dir).await?; - ctx.db.mark_downloaded(url, &path, got.bytes)?; + ctx.db.mark_downloaded(url, &path, got.bytes).await?; Ok((path, got.bytes)) } @@ -1513,7 +1513,7 @@ fn spawn_torrent(ctx: &Arc, feed_id: String, enclosure: i64, url: String, d let db = &ctx.db; match outcome { Ok((path, bytes)) => { - if let Err(e) = db.mark_downloaded(&url, &path, bytes) { + if let Err(e) = db.mark_downloaded(&url, &path, bytes).await { tracing::warn!(error = ?e, "could not record the finished torrent"); } ctx.out.emit(Event::DownloadDone { @@ -1526,7 +1526,7 @@ fn spawn_torrent(ctx: &Arc, feed_id: String, enclosure: i64, url: String, d } Err(e) => { let msg = format!("{e:#}"); - let _ = db.mark_enclosure(&url, "error", Some(&msg)); + let _ = db.mark_enclosure(&url, "error", Some(&msg)).await; ctx.out.emit(Event::DownloadError { feed: feed_id, enclosure, url, msg }); } } @@ -1539,7 +1539,7 @@ async fn download_one(ctx: &Arc, id: i64) -> Result<()> { let cfg = ctx.cfg(); let enc = ctx .db - .enclosure(id)? + .enclosure(id).await? .ok_or_else(|| anyhow::anyhow!("no enclosure {id}"))?; if enc.path.is_some() { return Ok(()); // Already here. @@ -1561,7 +1561,7 @@ async fn download_one(ctx: &Arc, id: i64) -> Result<()> { ctx.out.emit(Event::FeedStart { feed: enc.feed_id.clone() }); let is_torrent = download::looks_like_torrent(&enc.url, enc.mime.as_deref()); if is_torrent && cfg.torrent.enabled && ctx.detach_torrents { - ctx.db.mark_enclosure(&enc.url, "downloading", None)?; + ctx.db.mark_enclosure(&enc.url, "downloading", None).await?; spawn_torrent(ctx, enc.feed_id.clone(), enc.id, enc.url.clone(), dest_dir); return Ok(()); } @@ -1577,7 +1577,7 @@ async fn download_one(ctx: &Arc, id: i64) -> Result<()> { match result { Ok((path, bytes)) => { - ctx.db.mark_downloaded(&enc.url, &path, bytes)?; + ctx.db.mark_downloaded(&enc.url, &path, bytes).await?; ctx.out.emit(Event::DownloadDone { feed: enc.feed_id.clone(), enclosure: enc.id, @@ -1588,7 +1588,7 @@ async fn download_one(ctx: &Arc, id: i64) -> Result<()> { } Err(e) => { let msg = format!("{e:#}"); - ctx.db.mark_enclosure(&enc.url, "error", Some(&msg))?; + ctx.db.mark_enclosure(&enc.url, "error", Some(&msg)).await?; ctx.out.emit(Event::DownloadError { feed: enc.feed_id.clone(), enclosure: enc.id, @@ -1756,10 +1756,10 @@ mod tests { ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").unwrap(); ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").unwrap(); let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None }; - ctx.db.record_enclosure("has-file", "g1", &enc).unwrap(); - ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).unwrap(); + ctx.db.record_enclosure("has-file", "g1", &enc).await.unwrap(); + ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).await.unwrap(); - retire_group(&ctx, "parent").unwrap(); + retire_group(&ctx, "parent").await.unwrap(); let managed = ctx.db.managed_feeds().unwrap(); assert!(!managed.iter().any(|m| m.id == "empty"), "nothing downloaded, so it is forgotten"); @@ -1780,7 +1780,7 @@ mod tests { ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").unwrap(); ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).unwrap(); - assert_eq!(retire_stranded(&ctx).unwrap(), 2, "empty dropped, promoted unmanaged"); + assert_eq!(retire_stranded(&ctx).await.unwrap(), 2, "empty dropped, promoted unmanaged"); let managed: Vec = ctx.db.managed_feeds().unwrap().into_iter().map(|m| m.id).collect(); assert_eq!(managed, ["listed"], "a group still in config is left alone"); diff --git a/src/retention.rs b/src/retention.rs index 66ecf5b..a6e81e4 100644 --- a/src/retention.rs +++ b/src/retention.rs @@ -45,25 +45,25 @@ pub fn aged(candidates: &[Candidate], cutoff: i64) -> Vec { .collect() } -pub fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result { +pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result { let mut report = Report::default(); // Someone may have deleted a file by hand; the row must stop claiming it exists. - for (id, path) in db.missing_files()? { + for (id, path) in db.missing_files().await? { if !dry_run { - db.mark_reaped(id)?; + db.mark_reaped(id).await?; } tracing::debug!(path, "file gone, row reaped"); report.reconciled += 1; } - let candidates = db.reap_candidates()?; + let candidates = db.reap_candidates().await?; if cfg.general.max_age_days > 0 { let cutoff = now() - (cfg.general.max_age_days * 86_400) as i64; report.aged_out = aged(&candidates, cutoff); for c in &report.aged_out { - report.bytes_freed += remove(db, c, dry_run)?; + report.bytes_freed += remove(db, c, dry_run).await?; } if !dry_run { report.entries_pruned = db.prune_entries(cutoff)?; @@ -81,14 +81,14 @@ pub fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result { let total: u64 = remaining.iter().map(|c| c.bytes.max(0) as u64).sum(); report.over_quota = pick(&remaining, total, limit); for c in &report.over_quota { - report.bytes_freed += remove(db, c, dry_run)?; + report.bytes_freed += remove(db, c, dry_run).await?; } } Ok(report) } -fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result { +async fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result { if dry_run { return Ok(c.bytes.max(0) as u64); } @@ -100,7 +100,7 @@ fn remove(db: &Db, c: &Candidate, dry_run: bool) -> Result { tracing::warn!(path = c.path, error = %e, "could not delete"); return Ok(0); } - db.mark_reaped(c.id)?; + db.mark_reaped(c.id).await?; Ok(size) } @@ -176,7 +176,7 @@ mod tests { ) .unwrap(); - let got: Vec = db.reap_candidates().unwrap().iter().map(|c| c.id).collect(); + let got: Vec = db.reap_candidates().await.unwrap().iter().map(|c| c.id).collect(); assert_eq!( got, vec![4, 2, 3], diff --git a/src/web.rs b/src/web.rs index 67a36e8..1c897d5 100644 --- a/src/web.rs +++ b/src/web.rs @@ -1422,7 +1422,7 @@ async fn remove_feed( } cfg.save(&state.config_path)?; state.ctx.reload_cfg(&state.config_path)?; - crate::retire_group(&state.ctx, &id)?; + crate::retire_group(&state.ctx, &id).await?; Ok(StatusCode::NO_CONTENT) } @@ -1458,12 +1458,12 @@ async fn download_now( let enc = state .ctx .db - .enclosure(id)? + .enclosure(id).await? .ok_or_else(|| anyhow::anyhow!("no enclosure {id}"))?; if enc.path.is_some() { return Ok(StatusCode::NO_CONTENT); // Already here. } - state.ctx.db.requeue(id)?; + state.ctx.db.requeue(id).await?; state .cmds .send(Command::Download { enclosure: id }) @@ -1487,7 +1487,7 @@ async fn delete_file( let enc = state .ctx .db - .enclosure(id)? + .enclosure(id).await? .ok_or_else(|| anyhow::anyhow!("no enclosure {id}"))?; // There is one copy of the file: deleting it deletes everyone's. Say so before doing @@ -1519,7 +1519,7 @@ async fn delete_file( return Err(e.into()); } // The row survives as 'reaped', which is what stops the next scan re-downloading it. - state.ctx.db.mark_reaped(id)?; + state.ctx.db.mark_reaped(id).await?; state.events.send(Event::Reaped { path: enc.path.unwrap_or_default(), bytes: enc.length.unwrap_or(0).max(0) as u64, @@ -1573,7 +1573,7 @@ async fn media( Path(id): Path, req: Request, ) -> Response { - let Ok(Some(enc)) = state.ctx.db.enclosure(id) else { + let Ok(Some(enc)) = state.ctx.db.enclosure(id).await else { return (StatusCode::NOT_FOUND, "no such enclosure").into_response(); }; let Some(path) = enc.path else { @@ -1648,9 +1648,9 @@ async fn download_latest( Path(id): Path, Json(body): Json, ) -> Result, ApiError> { - let ids = state.ctx.db.undownloaded(&id, body.count.clamp(1, 100))?; + let ids = state.ctx.db.undownloaded(&id, body.count.clamp(1, 100)).await?; for enc in &ids { - state.ctx.db.requeue(*enc)?; + state.ctx.db.requeue(*enc).await?; state .cmds .send(Command::Download { enclosure: *enc }) From 611716d8b7ca2700187a43298020fbcc22a2d8bc Mon Sep 17 00:00:00 2001 From: rays Date: Fri, 18 Sep 2026 19:10:59 +0000 Subject: [PATCH 6/9] SeaORM: feeds and scanning; rusqlite gone The last nineteen functions move to SeaORM: recording feeds, items and enclosures, managed OPML feeds, folding WordPress's repeated files, and handing a Patreon creator's files to its shows. Two SQLite-only forms go: GLOB becomes a LIKE with the underscore escaped (broader, harmlessly: the fold still keys on `_=` and digits), and UPDATE OR IGNORE becomes an UPDATE ... WHERE NOT EXISTS. The two transactions are SeaORM transactions. With nothing left on it, rusqlite goes, with the SQL schema and migrate(). The entities are the schema: create_missing makes whatever tables and indexes a database lacks, from them, with CREATE ... IF NOT EXISTS. Production's schema already has every column migrate() added and none it dropped. Not SeaORM's schema sync, used until now: despite its docs it drops a unique index the entities do not describe, so it dropped users_name_lower on every open. Every `ipx` command then took a write lock, and against a daemon busy writing, `ipx status` -- the healthcheck -- failed 7 times in 15 where the old code failed none. Now 15 in 15, as before. On Postgres it would not have started. WAL is set only when a file is not already in it: setting it takes a lock that cannot wait out a busy daemon. Checked on copies of production: a forced scan of all 162 feeds against the real feeds with no database errors; the feed list, filters, sorts, search and the reaper's candidates against the old code on the same data, earlier in the branch. The column comments from the SQL schema move to the entities. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 3 + Cargo.lock | 50 --- Cargo.toml | 3 +- src/db.rs | 964 ++++++++++++++++++----------------------------- src/entity.rs | 28 +- src/main.rs | 134 +++---- src/retention.rs | 8 +- src/web.rs | 28 +- 8 files changed, 485 insertions(+), 733 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be53d5d..eb1b8da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- The database is reached through SeaORM, on the way to Postgres (issue #18); it is still the + same SQLite file, and nothing you see changes. A database from before 0.7 has to be opened by a + 0.7 release first, which brings its tables up to date. - A pinned item sits at the top of its list, above everything else in whatever order you sort by, and moves there the moment you pin it. Sorting by the pin column itself still goes both ways, and Currently Listening keeps its own order. diff --git a/Cargo.lock b/Cargo.lock index ca082d4..244d03b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1129,18 +1129,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastrand" version = "2.5.0" @@ -1848,7 +1836,6 @@ dependencies = [ "quick-xml 0.42.0", "reqwest", "rss", - "rusqlite", "sea-orm", "serde", "serde_json", @@ -3289,16 +3276,6 @@ dependencies = [ "libc", ] -[[package]] -name = "rsqlite-vfs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" -dependencies = [ - "hashbrown 0.16.1", - "thiserror 2.0.20", -] - [[package]] name = "rss" version = "2.1.1" @@ -3310,21 +3287,6 @@ dependencies = [ "quick-xml 0.41.0", ] -[[package]] -name = "rusqlite" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" -dependencies = [ - "bitflags 2.13.1", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", - "sqlite-wasm-rs", -] - [[package]] name = "rust_decimal" version = "1.43.0" @@ -3905,18 +3867,6 @@ dependencies = [ "lock_api", ] -[[package]] -name = "sqlite-wasm-rs" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" -dependencies = [ - "cc", - "js-sys", - "rsqlite-vfs", - "wasm-bindgen", -] - [[package]] name = "sqlx" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 847a9a0..cf598dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,7 @@ percent-encoding = "2.3.2" quick-xml = { version = "0.42.0", features = ["escape-html"] } reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "http2", "gzip", "stream", "json", "charset", "system-proxy"] } rss = "2.1.1" -rusqlite = { version = "0.39", features = ["bundled"] } -sea-orm = { version = "~2.0.3", default-features = false, features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "schema-sync", "sqlite-use-returning-for-3_35"] } +sea-orm = { version = "2.0.3", default-features = false, features = ["sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls", "macros", "with-json", "sqlite-use-returning-for-3_35"] } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "fs", "io-util", "net", "sync", "time", "signal"] } diff --git a/src/db.rs b/src/db.rs index dee1bd1..aa9f478 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,23 +1,17 @@ -//! SQLite state. Replaces the per-feed .ipxd plists, history.dat and qmcache.dat. +//! The database, through SeaORM: SQLite today, Postgres to come (issue #18). Replaces the +//! per-feed .ipxd plists, history.dat and qmcache.dat. use anyhow::{Context, Result}; -use crate::entity::{enclosures, sessions, subscriptions, users}; -use rusqlite::{Connection, OptionalExtension, params}; +use crate::entity::{enclosures, entries, feeds, sessions, subscriptions, users}; use sea_orm::sea_query::{Expr, Func}; use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, Set, Statement, }; use std::path::Path; -use std::sync::Mutex; -/// ponytail: one global connection mutex. Writes here are tiny and rare; move to a -/// spawn_blocking pool if a large feed count ever makes it contend. -/// -/// Mid-way through moving to SeaORM (issue #18): `orm` is the new connection, to the same -/// SQLite file, and functions move to it one at a time; `conn` goes when the last one has. +/// A pool of connections, where there was one connection behind a global lock. pub struct Db { - conn: Mutex, orm: sea_orm::DatabaseConnection, /// A test's database file, removed when the test is done with it. #[cfg(test)] @@ -35,26 +29,35 @@ impl Drop for Db { } } -/// Creates whatever the database is missing from the entities in `crate::entity`: tables, -/// columns, unique keys, foreign keys. It only ever adds. What an entity cannot say -- an -/// index on an expression, or on two columns -- follows as plain SQL both databases accept. +/// Creates whatever tables and indexes the database is missing, from the entities in +/// `crate::entity`, on SQLite or Postgres alike. It only ever creates: an existing table is left +/// as it is. What an entity cannot say -- an index on an expression, or on two columns -- +/// follows as plain SQL both databases accept. /// -/// ponytail: schema sync is experimental in SeaORM 2 and exempt from semver, hence the -/// `~2.0` pin in Cargo.toml; if it changes shape, move to sea-orm-migration files. -async fn sync(orm: &sea_orm::DatabaseConnection) -> Result<()> { +/// Not SeaORM's schema sync, which is experimental and, despite its docs, drops a unique index +/// the entities do not describe: it dropped users_name_lower on every open, so every `ipx` +/// command took a write lock, and the healthcheck's `ipx status` timed out behind a busy daemon. +/// On Postgres it would have failed outright, dropping that index as a constraint. +/// +/// ponytail: tables only, no columns. A column added to an existing table needs its own ALTER +/// here, as the old migrate() did for SQLite, or sea-orm-migration once there are several. A +/// database from before 0.7 took its last columns from that migrate(), so it upgrades through +/// a 0.7 release first. +async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> { use crate::entity::*; - use sea_orm::ConnectionTrait; - orm.get_schema_builder() - .register(feeds::Entity) - .register(entries::Entity) - .register(enclosures::Entity) - .register(users::Entity) - .register(subscriptions::Entity) - .register(entry_state::Entity) - .register(sessions::Entity) - .sync(orm) - .await - .context("creating the schema")?; + use sea_orm::{ConnectionTrait, Schema}; + let schema = Schema::new(orm.get_database_backend()); + for mut table in [ + schema.create_table_from_entity(feeds::Entity), + schema.create_table_from_entity(entries::Entity), + schema.create_table_from_entity(enclosures::Entity), + schema.create_table_from_entity(users::Entity), + schema.create_table_from_entity(subscriptions::Entity), + schema.create_table_from_entity(entry_state::Entity), + schema.create_table_from_entity(sessions::Entity), + ] { + orm.execute(table.if_not_exists()).await.context("creating the schema")?; + } for sql in [ "CREATE INDEX IF NOT EXISTS enclosures_entry ON enclosures (feed_id, guid)", "CREATE UNIQUE INDEX IF NOT EXISTS users_name_lower ON users (lower(name))", @@ -64,6 +67,8 @@ async fn sync(orm: &sea_orm::DatabaseConnection) -> Result<()> { Ok(()) } +/// sqlx's defaults for SQLite are what ipx wants: foreign keys on, and a five-second wait for a +/// lock, which is what rusqlite was set to. async fn connect(path: &Path) -> Result { let mut opts = sea_orm::ConnectOptions::new(format!("sqlite://{}?mode=rwc", path.display())); opts.sqlx_logging(false); @@ -72,116 +77,6 @@ async fn connect(path: &Path) -> Result { .with_context(|| format!("opening {}", path.display())) } -const SCHEMA: &str = " -CREATE TABLE IF NOT EXISTS feeds ( - id TEXT PRIMARY KEY, - url TEXT NOT NULL, - title TEXT, - image TEXT, - -- The channel's first , for the Directory. - category TEXT, - etag TEXT, - last_modified TEXT, - last_checked INTEGER, - ttl_mins INTEGER, - last_error TEXT, - -- When the current run of failures began; NULL while the feed is healthy. Kept - -- through repeated failures so the UI can tell a blip (macmanx: failed once, fine an - -- hour later) from a feed that has been down for a day. - error_since INTEGER, - -- Came from a subscribed OPML that no longer lists it, but has downloads, so kept. - orphaned INTEGER NOT NULL DEFAULT 0, - -- The OPML subscription this feed came from. - group_id TEXT, - -- 1 = derived from an OPML and not written to config.toml. Writing 80-odd generated - -- entries into a hand-edited file made it unreadable; the OPML is the source of - -- truth, so they are re-derived instead. Customising one promotes it to config. - managed INTEGER NOT NULL DEFAULT 0 -); - -CREATE TABLE IF NOT EXISTS entries ( - feed_id TEXT NOT NULL, - guid TEXT NOT NULL, - title TEXT, - link TEXT, - published INTEGER, - description TEXT, - first_seen INTEGER NOT NULL, - image TEXT, - duration INTEGER, - episode INTEGER, - season INTEGER, - PRIMARY KEY (feed_id, guid) -); - --- url is UNIQUE: this is the dedupe key, and it subsumes the old history.dat pickle. --- A reaped file keeps its row with path = NULL and state = 'reaped', so a purged --- episode is never fetched a second time. -CREATE TABLE IF NOT EXISTS enclosures ( - id INTEGER PRIMARY KEY, - feed_id TEXT NOT NULL, - guid TEXT NOT NULL, - url TEXT NOT NULL UNIQUE, - mime TEXT, - length INTEGER, - path TEXT, - state TEXT NOT NULL, - bytes_done INTEGER NOT NULL DEFAULT 0, - downloaded_at INTEGER, - last_error TEXT -); - -CREATE INDEX IF NOT EXISTS enclosures_entry ON enclosures (feed_id, guid); - --- pass_hash is NULL for someone who only ever arrives through the proxy: there is no --- password to check, and leaving it empty is not the same as leaving it unset. -CREATE TABLE IF NOT EXISTS users ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL UNIQUE COLLATE NOCASE, - pass_hash TEXT, - is_admin INTEGER NOT NULL DEFAULT 0, - -- For whoever maintains the server. NULL where it is not known. - created INTEGER, - last_login INTEGER, - -- The theme chosen in Settings, and light, dark or auto. NULL until one is chosen. - theme TEXT, - theme_mode TEXT -); - --- What one person wants from a feed. The feed, its items and its files are shared; this --- is the part that is not. NULL in a column means: follow the feed's own setting. -CREATE TABLE IF NOT EXISTS subscriptions ( - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - feed_id TEXT NOT NULL, - keywords TEXT, - auto_download INTEGER, - allow_explicit INTEGER, - max_new_per_check INTEGER, - -- Pinned to the top of this person's feed list, a feed inside a folder included. - pinned INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (user_id, feed_id) -); - --- Read, kept and how far in. One row per person per item, created on first touch; --- an item nobody has touched has no row at all, which is what unread means. -CREATE TABLE IF NOT EXISTS entry_state ( - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - feed_id TEXT NOT NULL, - guid TEXT NOT NULL, - read INTEGER NOT NULL DEFAULT 0, - flagged INTEGER NOT NULL DEFAULT 0, - position INTEGER NOT NULL DEFAULT 0, - -- The length this person's player measured, beside the position it is measured against. - duration INTEGER, - PRIMARY KEY (user_id, feed_id, guid) -); - -CREATE TABLE IF NOT EXISTS sessions ( - token TEXT PRIMARY KEY, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - seen INTEGER NOT NULL -); -"; /// One person's wants for one feed. `None` in a field means the feed's own setting stands. #[derive(Debug, Clone, Default)] @@ -261,62 +156,6 @@ pub struct Managed { pub group_id: String, } -/// Adds the columns later versions introduced and drops the ones they retired. CREATE TABLE IF -/// NOT EXISTS leaves a table that already exists alone, so an installed database needs both done -/// explicitly. Columns from before 0.3.0, the oldest version an upgrade may start from, need no -/// entry. -fn migrate(conn: &Connection) -> Result<()> { - let wanted: &[(&str, &str, &str)] = &[ - // For whoever maintains the server. An audit dropped `created` as unread on 2026-09-12, - // and it came back the same day with `last_login` beside it. - ("users", "created", "INTEGER"), - ("users", "last_login", "INTEGER"), - ("feeds", "error_since", "INTEGER"), - ("feeds", "category", "TEXT"), - ("entry_state", "duration", "INTEGER"), - // Kept per account so a theme follows you to another browser; it was in localStorage. - ("users", "theme", "TEXT"), - ("users", "theme_mode", "TEXT"), - ("subscriptions", "pinned", "INTEGER NOT NULL DEFAULT 0"), - ]; - let retired: &[(&str, &str)] = &[ - // Read state from before accounts, long since moved to entry_state. Two bugs came from - // queries still reading these after they stopped meaning anything. - ("entries", "read"), - ("entries", "flagged"), - ("entries", "position"), - // Written by every insert and read by nothing. - ("subscriptions", "created"), - ("sessions", "created"), - ]; - let has = |table: &str, column: &str| -> Result { - let names: Vec = conn - .prepare(&format!("PRAGMA table_info({table})"))? - .query_map([], |r| r.get(1))? - .collect::>()?; - Ok(names.iter().any(|c| c == column)) - }; - for (table, column, ty) in wanted { - if !has(table, column)? { - tracing::info!(table, column, "adding column"); - conn.execute_batch(&format!("ALTER TABLE {table} ADD COLUMN {column} {ty}"))?; - // A category is only read from a 200, and a feed with a validator mostly gets a - // 304, so most would never pick one up until the publisher changed something. - // Dropping the validators once makes each re-read on its normal schedule; leaving - // last_checked alone, unlike clear_validators, keeps them from all coming due at once. - if (*table, *column) == ("feeds", "category") { - conn.execute_batch("UPDATE feeds SET etag = NULL, last_modified = NULL")?; - } - } - } - for (table, column) in retired { - if has(table, column)? { - tracing::info!(table, column, "dropping column"); - conn.execute_batch(&format!("ALTER TABLE {table} DROP COLUMN {column}"))?; - } - } - Ok(()) -} /// What `ipx list` shows next to each configured feed. #[derive(Debug, Default)] @@ -341,17 +180,16 @@ impl Db { std::fs::create_dir_all(dir) .with_context(|| format!("creating {}", dir.display()))?; } - let conn = Connection::open(path) - .with_context(|| format!("opening {}", path.display()))?; - conn.pragma_update(None, "journal_mode", "WAL")?; - conn.pragma_update(None, "foreign_keys", "ON")?; - conn.pragma_update(None, "busy_timeout", 5000)?; - conn.execute_batch(SCHEMA).context("creating schema")?; - migrate(&conn).context("migrating schema")?; let orm = connect(path).await?; - sync(&orm).await?; + // WAL, so the healthcheck's `ipx status` reads while the daemon writes. It is a setting of + // the file, kept once made, and making it takes a lock that cannot wait out a busy + // daemon, so it is made only when the file is not already in WAL. + let mode = orm.query_one_raw(Statement::from_string(orm.get_database_backend(), "PRAGMA journal_mode")).await?; + if mode.and_then(|r| r.try_get_by_index::(0).ok()).as_deref() != Some("wal") { + orm.execute_unprepared("PRAGMA journal_mode = WAL").await.context("switching to WAL")?; + } + create_missing(&orm).await?; Ok(Self { - conn: Mutex::new(conn), orm, #[cfg(test)] tmp: None, @@ -359,8 +197,7 @@ impl Db { } /// A fresh database for a test, in a file of its own: two connections to one ":memory:" - /// would be two databases. Its schema comes from the entities alone (`sync`), not SCHEMA, - /// so every test also checks that the entities describe everything the queries need. + /// would be two databases. Its schema comes from the entities, as a real one's does. #[cfg(test)] pub async fn memory() -> Result { static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); @@ -370,53 +207,46 @@ impl Db { N.fetch_add(1, std::sync::atomic::Ordering::Relaxed) )); let orm = connect(&path).await?; - sync(&orm).await?; - let conn = Connection::open(&path)?; - conn.pragma_update(None, "foreign_keys", "ON")?; - conn.pragma_update(None, "busy_timeout", 5000)?; - Ok(Self { conn: Mutex::new(conn), orm, tmp: Some(path) }) + create_missing(&orm).await?; + Ok(Self { orm, tmp: Some(path) }) } #[cfg(test)] - pub fn exec_for_test(&self, sql: &str) -> Result<()> { - self.conn.lock().unwrap().execute_batch(sql)?; + /// The first column of every row, for a test to check what a query left behind. + #[cfg(test)] + pub async fn i64s_for_test(&self, sql: &str) -> Vec { + self.rows(sql, vec![]).await.unwrap().iter().map(|r| r.try_get_by_index(0).unwrap()).collect() + } + + #[cfg(test)] + pub async fn strings_for_test(&self, sql: &str) -> Vec { + self.rows(sql, vec![]).await.unwrap().iter().map(|r| r.try_get_by_index(0).unwrap()).collect() + } + + #[cfg(test)] + pub async fn exec_for_test(&self, sql: &str) -> Result<()> { + self.orm.execute_unprepared(sql).await?; Ok(()) } - pub fn feed_summary(&self, feed_id: &str) -> Result { - let conn = self.conn.lock().unwrap(); - let mut sum: FeedSummary = conn - .query_row( - "SELECT title, image, last_checked, last_error, coalesce(orphaned, 0), error_since, - category - FROM feeds WHERE id = ?1", - [feed_id], - |r| { - Ok(FeedSummary { - title: r.get(0)?, - image: r.get(1)?, - last_checked: r.get(2)?, - last_error: r.get(3)?, - orphaned: r.get::<_, i64>(4)? != 0, - error_since: r.get(5)?, - category: r.get(6)?, - ..Default::default() - }) - }, - ) - .optional()? + pub async fn feed_summary(&self, feed_id: &str) -> Result { + let mut sum = feeds::Entity::find_by_id(feed_id.to_owned()) + .one(&self.orm) + .await? + .map(|f| FeedSummary { + title: f.title, + image: f.image, + last_checked: f.last_checked, + last_error: f.last_error, + orphaned: f.orphaned, + error_since: f.error_since, + category: f.category, + ..Default::default() + }) .unwrap_or_default(); - - sum.entries = conn.query_row( - "SELECT count(*) FROM entries WHERE feed_id = ?1", - [feed_id], - |r| r.get(0), - )?; - sum.downloaded = conn.query_row( - "SELECT count(*) FROM enclosures WHERE feed_id = ?1 AND path IS NOT NULL", - [feed_id], - |r| r.get(0), - )?; + sum.entries = + entries::Entity::find().filter(entries::Column::FeedId.eq(feed_id)).count(&self.orm).await? as i64; + sum.downloaded = self.downloaded_count(feed_id).await?; Ok(sum) } } @@ -432,27 +262,22 @@ pub struct HttpState { } impl Db { - pub fn http_state(&self, feed_id: &str) -> Result { - let conn = self.conn.lock().unwrap(); - Ok(conn - .query_row( - "SELECT etag, last_modified, last_checked, ttl_mins FROM feeds WHERE id = ?1", - [feed_id], - |r| { - Ok(HttpState { - etag: r.get(0)?, - last_modified: r.get(1)?, - last_checked: r.get(2)?, - ttl_mins: r.get::<_, Option>(3)?.map(|t| t.max(0) as u64), - }) - }, - ) - .optional()? + pub async fn http_state(&self, feed_id: &str) -> Result { + Ok(feeds::Entity::find_by_id(feed_id.to_owned()) + .one(&self.orm) + .await? + .map(|f| HttpState { + etag: f.etag, + last_modified: f.last_modified, + last_checked: f.last_checked, + ttl_mins: f.ttl_mins.map(|t| t.max(0) as u64), + }) .unwrap_or_default()) } /// Upsert after a successful poll. Clears any previous error. - pub fn record_feed( + #[allow(clippy::too_many_arguments)] + pub async fn record_feed( &self, feed_id: &str, url: &str, @@ -463,13 +288,13 @@ impl Db { image: Option<&str>, category: Option<&str>, ) -> Result<()> { - let conn = self.conn.lock().unwrap(); // category is taken as it comes, unlike title and image: a show that leaves a category // should leave the Directory's chip too. - conn.execute( + let o = |v: Option<&str>| sea_orm::Value::from(v.map(str::to_owned)); + self.exec( "INSERT INTO feeds (id, url, title, etag, last_modified, last_checked, ttl_mins, last_error, image, category) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, NULL, ?8, ?9) - ON CONFLICT(id) DO UPDATE SET + VALUES ($1, $2, $3, $4, $5, $6, $7, NULL, $8, $9) + ON CONFLICT (id) DO UPDATE SET url = excluded.url, title = coalesce(excluded.title, feeds.title), etag = excluded.etag, @@ -480,78 +305,106 @@ impl Db { category = excluded.category, last_error = NULL, error_since = NULL", - rusqlite::params![feed_id, url, title, etag, last_modified, now(), ttl_mins.map(|t| t as i64), image, category], - )?; + vec![ + feed_id.into(), + url.into(), + o(title), + o(etag), + o(last_modified), + now().into(), + ttl_mins.map(|t| t as i64).into(), + o(image), + o(category), + ], + ) + .await?; Ok(()) } /// 304, or any other poll that produced no new data: only the clock moves. - pub fn touch_feed(&self, feed_id: &str, url: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "INSERT INTO feeds (id, url, last_checked) VALUES (?1, ?2, ?3) - ON CONFLICT(id) DO UPDATE SET last_checked = excluded.last_checked, + pub async fn touch_feed(&self, feed_id: &str, url: &str) -> Result<()> { + self.exec( + "INSERT INTO feeds (id, url, last_checked) VALUES ($1, $2, $3) + ON CONFLICT (id) DO UPDATE SET last_checked = excluded.last_checked, last_error = NULL, error_since = NULL", - rusqlite::params![feed_id, url, now()], - )?; + vec![feed_id.into(), url.into(), now().into()], + ) + .await?; Ok(()) } /// Forgets the cached ETag/Last-Modified. Those validators belong to the old URL, so /// keeping them across a URL change could produce a bogus 304 against the new one. - pub fn clear_validators(&self, feed_id: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE feeds SET etag = NULL, last_modified = NULL, last_checked = NULL WHERE id = ?1", - [feed_id], - )?; + pub async fn clear_validators(&self, feed_id: &str) -> Result<()> { + self.exec( + "UPDATE feeds SET etag = NULL, last_modified = NULL, last_checked = NULL WHERE id = $1", + vec![feed_id.into()], + ) + .await?; Ok(()) } - pub fn set_feed_error(&self, feed_id: &str, url: &str, msg: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - let now = now(); - conn.execute( + pub async fn set_feed_error(&self, feed_id: &str, url: &str, msg: &str) -> Result<()> { + self.exec( "INSERT INTO feeds (id, url, last_checked, last_error, error_since) - VALUES (?1, ?2, ?3, ?4, ?3) - ON CONFLICT(id) DO UPDATE SET + VALUES ($1, $2, $3, $4, $3) + ON CONFLICT (id) DO UPDATE SET last_checked = excluded.last_checked, last_error = excluded.last_error, error_since = coalesce(feeds.error_since, excluded.error_since)", - rusqlite::params![feed_id, url, now, msg], - )?; + vec![feed_id.into(), url.into(), now().into(), msg.into()], + ) + .await?; Ok(()) } /// Returns true when this entry had not been seen before. /// - pub fn record_entry(&self, feed_id: &str, e: &crate::feed::Entry) -> Result { - let conn = self.conn.lock().unwrap(); - let inserted = conn.execute( - "INSERT OR IGNORE INTO entries - (feed_id, guid, title, link, published, description, first_seen, - image, duration, episode, season) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - rusqlite::params![ - feed_id, e.guid, e.title, e.link, e.published, e.description, now(), - e.image, e.duration, e.episode, e.season - ], - )?; - if inserted == 0 { - conn.execute( - "UPDATE entries SET - title = coalesce(?3, title), - description = coalesce(?4, description), - image = coalesce(?5, image), - duration = coalesce(?6, duration), - episode = coalesce(?7, episode), - season = coalesce(?8, season) - WHERE feed_id = ?1 AND guid = ?2", - rusqlite::params![ - feed_id, e.guid, e.title, e.description, - e.image, e.duration, e.episode, e.season + pub async fn record_entry(&self, feed_id: &str, e: &crate::feed::Entry) -> Result { + let inserted = self + .exec( + "INSERT INTO entries + (feed_id, guid, title, link, published, description, first_seen, + image, duration, episode, season) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT DO NOTHING", + vec![ + feed_id.into(), + e.guid.clone().into(), + e.title.clone().into(), + e.link.clone().into(), + e.published.into(), + e.description.clone().into(), + now().into(), + e.image.clone().into(), + e.duration.into(), + e.episode.into(), + e.season.into(), ], - )?; + ) + .await?; + if inserted == 0 { + self.exec( + "UPDATE entries SET + title = coalesce($3, title), + description = coalesce($4, description), + image = coalesce($5, image), + duration = coalesce($6, duration), + episode = coalesce($7, episode), + season = coalesce($8, season) + WHERE feed_id = $1 AND guid = $2", + vec![ + feed_id.into(), + e.guid.clone().into(), + e.title.clone().into(), + e.description.clone().into(), + e.image.clone().into(), + e.duration.into(), + e.episode.into(), + e.season.into(), + ], + ) + .await?; } Ok(inserted == 1) } @@ -710,31 +563,33 @@ impl Db { /// Old entries that never had a file, or no longer have one. Enclosure rows stay -- /// they are the dedupe history. - pub fn prune_entries(&self, older_than: i64) -> Result { - let conn = self.conn.lock().unwrap(); - let n = conn.execute( - "DELETE FROM entries - WHERE coalesce(published, first_seen) < ?1 - AND NOT EXISTS ( - SELECT 1 FROM enclosures e - WHERE e.feed_id = entries.feed_id AND e.guid = entries.guid - AND e.path IS NOT NULL) - -- Starred by anyone keeps it, the same rule the reaper follows. - AND NOT EXISTS ( - SELECT 1 FROM entry_state s - WHERE s.feed_id = entries.feed_id AND s.guid = entries.guid - AND s.flagged = 1)", - [older_than], - )?; + pub async fn prune_entries(&self, older_than: i64) -> Result { + let n = self + .exec( + "DELETE FROM entries + WHERE coalesce(published, first_seen) < $1 + AND NOT EXISTS ( + SELECT 1 FROM enclosures e + WHERE e.feed_id = entries.feed_id AND e.guid = entries.guid + AND e.path IS NOT NULL) + -- Starred by anyone keeps it, the same rule the reaper follows. + AND NOT EXISTS ( + SELECT 1 FROM entry_state s + WHERE s.feed_id = entries.feed_id AND s.guid = entries.guid + AND s.flagged)", + vec![older_than.into()], + ) + .await?; // Whatever went takes everyone's read state with it, rather than leaving rows // pointing at an item that no longer exists. - conn.execute( + self.exec( "DELETE FROM entry_state WHERE NOT EXISTS ( SELECT 1 FROM entries e WHERE e.feed_id = entry_state.feed_id AND e.guid = entry_state.guid)", - [], - )?; - Ok(n) + vec![], + ) + .await?; + Ok(n as usize) } } @@ -1458,64 +1313,58 @@ impl Db { /// Names a feed without touching its conditional-GET validators. An OPML subscription /// takes its name from the document's own . - pub fn set_title(&self, feed_id: &str, title: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "UPDATE feeds SET title = ?2 WHERE id = ?1 AND coalesce(title, '') != ?2", - rusqlite::params![feed_id, title], - )?; + pub async fn set_title(&self, feed_id: &str, title: &str) -> Result<()> { + self.exec( + "UPDATE feeds SET title = $2 WHERE id = $1 AND coalesce(title, '') <> $2", + vec![feed_id.into(), title.into()], + ) + .await?; Ok(()) } /// Records a feed that came from an OPML. Its settings are the parent's; only what /// identifies it is stored. - pub fn upsert_managed( - &self, - id: &str, - url: &str, - title: &str, - group_id: &str, - ) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( + pub async fn upsert_managed(&self, id: &str, url: &str, title: &str, group_id: &str) -> Result<()> { + self.exec( "INSERT INTO feeds (id, url, title, group_id, managed, orphaned) - VALUES (?1, ?2, ?3, ?4, 1, 0) - ON CONFLICT(id) DO UPDATE SET + VALUES ($1, $2, $3, $4, true, false) + ON CONFLICT (id) DO UPDATE SET url = excluded.url, title = coalesce(feeds.title, excluded.title), group_id = excluded.group_id, - managed = 1, - orphaned = 0", - rusqlite::params![id, url, title, group_id], - )?; + managed = true, + orphaned = false", + vec![id.into(), url.into(), title.into(), group_id.into()], + ) + .await?; Ok(()) } /// Every feed derived from an OPML, whichever group. - pub fn managed_feeds(&self) -> Result<Vec<Managed>> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( + pub async fn managed_feeds(&self) -> Result<Vec<Managed>> { + self.rows( "SELECT id, url, title, group_id FROM feeds - WHERE managed = 1 AND group_id IS NOT NULL ORDER BY coalesce(title, id)", - )?; - Ok(stmt - .query_map([], |r| { - Ok(Managed { - id: r.get(0)?, - url: r.get(1)?, - title: r.get(2)?, - group_id: r.get(3)?, - }) - })? - .collect::<rusqlite::Result<Vec<_>>>()?) + WHERE managed AND group_id IS NOT NULL ORDER BY coalesce(title, id)", + vec![], + ) + .await? + .iter() + .map(|r| { + Ok(Managed { + id: r.try_get("", "id")?, + url: r.try_get("", "url")?, + title: r.try_get("", "title")?, + group_id: r.try_get("", "group_id")?, + }) + }) + .collect() } /// Forgets a derived feed entirely. Only for one with nothing downloaded. - pub fn drop_managed(&self, id: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("DELETE FROM feeds WHERE id = ?1 AND managed = 1", [id])?; - conn.execute("DELETE FROM entries WHERE feed_id = ?1", [id])?; - conn.execute("DELETE FROM enclosures WHERE feed_id = ?1 AND path IS NULL", [id])?; + pub async fn drop_managed(&self, id: &str) -> Result<()> { + self.exec("DELETE FROM feeds WHERE id = $1 AND managed", vec![id.into()]).await?; + self.exec("DELETE FROM entries WHERE feed_id = $1", vec![id.into()]).await?; + self.exec("DELETE FROM enclosures WHERE feed_id = $1 AND path IS NULL", vec![id.into()]).await?; Ok(()) } @@ -1523,24 +1372,35 @@ impl Db { /// WordPress's numbered player URLs (`feed::same_file_key`). The first is the one the parser /// keeps, so it keeps its row, taking a repeat's file if it has none of its own; the repeats' /// rows go. Returns how many went and the copies left spare, for the caller to delete. - pub fn merge_repeated_enclosures(&self, key: impl Fn(&str) -> String) -> Result<(usize, Vec<String>)> { + pub async fn merge_repeated_enclosures(&self, key: impl Fn(&str) -> String) -> Result<(usize, Vec<String>)> { + use sea_orm::TransactionTrait; use std::collections::hash_map::Entry; - let mut conn = self.conn.lock().unwrap(); - let tx = conn.transaction()?; - let rows: Vec<(i64, String, String, String, Option<String>)> = { - let mut stmt = tx.prepare( - "SELECT id, feed_id, guid, url, path FROM enclosures + let backend = self.orm.get_database_backend(); + let tx = self.orm.begin().await?; + // Items with a URL carrying WordPress's `_=` parameter. A LIKE, with the underscore + // escaped, where SQLite had GLOB '*[?&]_=[0-9]*', which Postgres lacks. It lets through + // `_=` without a number too, which is harmless: `key` only folds `_=` and digits. + let rows = tx + .query_all_raw(Statement::from_string( + backend, + r"SELECT id, feed_id, guid, url, path FROM enclosures WHERE (feed_id, guid) IN - (SELECT feed_id, guid FROM enclosures WHERE url GLOB '*[?&]_=[0-9]*') + (SELECT feed_id, guid FROM enclosures + WHERE url LIKE '%?\_=%' ESCAPE '\' OR url LIKE '%&\_=%' ESCAPE '\') ORDER BY id", - )?; - stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)))? - .collect::<rusqlite::Result<_>>()? - }; + )) + .await?; // The first row of each file, and whether it has the file on disk yet. let mut first: std::collections::HashMap<(String, String, String), (i64, bool)> = Default::default(); let (mut gone, mut spare) = (0, vec![]); - for (id, feed, guid, url, path) in rows { + for r in rows { + let (id, feed, guid, url, path): (i64, String, String, String, Option<String>) = ( + r.try_get("", "id")?, + r.try_get("", "feed_id")?, + r.try_get("", "guid")?, + r.try_get("", "url")?, + r.try_get("", "path")?, + ); match first.entry((feed, guid, key(&url))) { Entry::Vacant(v) => { v.insert((id, path.is_some())); @@ -1553,103 +1413,115 @@ impl Db { } else { // The only copy is the repeat's: Rands' episode 97 was downloaded // under its ?_=2 URL alone. - tx.execute( + tx.execute_raw(Statement::from_sql_and_values( + backend, "UPDATE enclosures SET (path, state, bytes_done, downloaded_at) = - (SELECT path, state, bytes_done, downloaded_at FROM enclosures WHERE id = ?2) - WHERE id = ?1", - params![*keep, id], - )?; + (SELECT path, state, bytes_done, downloaded_at FROM enclosures WHERE id = $2) + WHERE id = $1", + vec![(*keep).into(), id.into()], + )) + .await?; *has = true; } } - tx.execute("DELETE FROM enclosures WHERE id = ?1", [id])?; + tx.execute_raw(Statement::from_sql_and_values( + backend, + "DELETE FROM enclosures WHERE id = $1", + vec![id.into()], + )) + .await?; gone += 1; } } } - tx.commit()?; + tx.commit().await?; Ok((gone, spare)) } /// Stops treating a feed as derived, because it now has its own config entry. - pub fn unmanage(&self, id: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("UPDATE feeds SET managed = 0 WHERE id = ?1", [id])?; + pub async fn unmanage(&self, id: &str) -> Result<()> { + self.exec("UPDATE feeds SET managed = false WHERE id = $1", vec![id.into()]).await?; Ok(()) } /// Empties a feed of its items, leaving its files alone. - pub fn clear_entries(&self, feed_id: &str) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute("DELETE FROM entries WHERE feed_id = ?1", [feed_id])?; + pub async fn clear_entries(&self, feed_id: &str) -> Result<()> { + entries::Entity::delete_many().filter(entries::Column::FeedId.eq(feed_id)).exec(&self.orm).await?; Ok(()) } /// Every feed the database holds rows for, as (id, url), removed ones included. - pub fn feed_urls(&self) -> Result<Vec<(String, String)>> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare("SELECT id, coalesce(url, '') FROM feeds")?; - let out = stmt - .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))? - .collect::<rusqlite::Result<_>>()?; - Ok(out) + pub async fn feed_urls(&self) -> Result<Vec<(String, String)>> { + Ok(feeds::Entity::find().all(&self.orm).await?.into_iter().map(|f| (f.id, f.url)).collect()) } /// Hands a feed in a group the enclosures its parent holds, as (guid, url), with everyone's /// read state for them. A Patreon creator read as one feed before it was split into shows /// owns every show's files, and `enclosures.url` is unique, so without this each show would /// list its items with nothing to play. - pub fn adopt(&self, parent: &str, child: &str, listed: &[(&str, &str)]) -> Result<()> { - let mut conn = self.conn.lock().unwrap(); - let holds: bool = conn.query_row( - "SELECT EXISTS (SELECT 1 FROM enclosures WHERE feed_id = ?1)", - [parent], - |r| r.get(0), - )?; + pub async fn adopt(&self, parent: &str, child: &str, listed: &[(&str, &str)]) -> Result<()> { + use sea_orm::TransactionTrait; + let holds = enclosures::Entity::find() + .filter(enclosures::Column::FeedId.eq(parent)) + .count(&self.orm) + .await? + > 0; if !holds { return Ok(()); // An OPML, or a creator already shared out. } - let tx = conn.transaction()?; + let backend = self.orm.get_database_backend(); + let tx = self.orm.begin().await?; for &(guid, url) in listed { - let moved = tx.execute( - "UPDATE enclosures SET feed_id = ?3, guid = ?4 WHERE url = ?1 AND feed_id = ?2", - params![url, parent, child, guid], - )?; + let moved = tx + .execute_raw(Statement::from_sql_and_values( + backend, + "UPDATE enclosures SET feed_id = $3, guid = $4 WHERE url = $1 AND feed_id = $2", + vec![url.into(), parent.into(), child.into(), guid.into()], + )) + .await? + .rows_affected(); if moved == 1 { - // Patreon gives a post the same guid in every feed it appears in. - tx.execute( - "UPDATE OR IGNORE entry_state SET feed_id = ?2 WHERE feed_id = ?1 AND guid = ?3", - params![parent, child, guid], - )?; + // Patreon gives a post the same guid in every feed it appears in. Someone who + // already has a row for it under the child keeps that one; SQLite's UPDATE OR + // IGNORE did this, and Postgres has no such thing. + tx.execute_raw(Statement::from_sql_and_values( + backend, + "UPDATE entry_state SET feed_id = $2 WHERE feed_id = $1 AND guid = $3 + AND NOT EXISTS (SELECT 1 FROM entry_state t + WHERE t.user_id = entry_state.user_id + AND t.feed_id = $2 AND t.guid = $3)", + vec![parent.into(), child.into(), guid.into()], + )) + .await?; } } - tx.commit()?; + tx.commit().await?; Ok(()) } /// A feed's enclosures skipped by one of its filters, by URL, with the reason: the verdicts a /// change of settings can overturn. A torrent held back while torrents are off is not a /// filter's call. - pub fn skipped_by_filter(&self, feed_id: &str) -> Result<std::collections::HashMap<String, String>> { - let conn = self.conn.lock().unwrap(); - let mut stmt = conn.prepare( + pub async fn skipped_by_filter(&self, feed_id: &str) -> Result<std::collections::HashMap<String, String>> { + self.rows( "SELECT url, last_error FROM enclosures - WHERE feed_id = ?1 AND state = 'skipped' AND last_error IS NOT NULL - AND last_error != 'torrents disabled'", - )?; - let out = stmt - .query_map([feed_id], |r| Ok((r.get(0)?, r.get(1)?)))? - .collect::<rusqlite::Result<_>>()?; - Ok(out) + WHERE feed_id = $1 AND state = 'skipped' AND last_error IS NOT NULL + AND last_error <> 'torrents disabled'", + vec![feed_id.into()], + ) + .await? + .iter() + .map(|r| Ok((r.try_get("", "url")?, r.try_get("", "last_error")?))) + .collect() } - pub fn set_orphaned(&self, feed_id: &str, on: bool) -> Result<()> { - let conn = self.conn.lock().unwrap(); - conn.execute( - "INSERT INTO feeds (id, url, orphaned) VALUES (?1, '', ?2) - ON CONFLICT(id) DO UPDATE SET orphaned = excluded.orphaned", - rusqlite::params![feed_id, on as i64], - )?; + pub async fn set_orphaned(&self, feed_id: &str, on: bool) -> Result<()> { + self.exec( + "INSERT INTO feeds (id, url, orphaned) VALUES ($1, '', $2) + ON CONFLICT (id) DO UPDATE SET orphaned = excluded.orphaned", + vec![feed_id.into(), on.into()], + ) + .await?; Ok(()) } @@ -1704,50 +1576,22 @@ mod tests { (5,'f','c','https://x/c.mp3?_=1','/d/c.mp3','done'), (6,'f','d','https://x/d1.mp3?_=1','/d/d1.mp3','done'), (7,'f','d','https://x/d2.mp3?_=2','/d/d2.mp3','done');", - ) + ).await .unwrap(); let key = crate::feed::same_file_key; - assert_eq!(db.merge_repeated_enclosures(key).unwrap(), (2, vec!["/d/a.mp3".to_string()])); - { - let conn = db.conn.lock().unwrap(); - let ids: Vec<i64> = conn - .prepare("SELECT id FROM enclosures ORDER BY id") - .unwrap() - .query_map([], |r| r.get(0)) - .unwrap() - .collect::<rusqlite::Result<_>>() - .unwrap(); - assert_eq!(ids, [1, 3, 5, 6, 7], "a lone ?_=1 and two different files stay"); - let (path, state): (String, String) = - conn.query_row("SELECT path, state FROM enclosures WHERE id = 3", [], |r| Ok((r.get(0)?, r.get(1)?))).unwrap(); - assert_eq!((path.as_str(), state.as_str()), ("/d/b.mp3", "done"), "the only copy moves, not deleted"); - } - assert_eq!(db.merge_repeated_enclosures(key).unwrap(), (0, vec![]), "and only once"); - } - - #[test] - fn adding_category_drops_validators_once_and_keeps_the_schedule() { - let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch(SCHEMA).unwrap(); - // A database from before the column, holding a feed that would answer 304. - conn.execute_batch( - "ALTER TABLE feeds DROP COLUMN category; - INSERT INTO feeds (id, url, etag, last_modified, last_checked) VALUES ('f','u','e','lm',5);", - ) - .unwrap(); - let row = |conn: &Connection| -> (Option<String>, Option<String>, Option<i64>) { - conn.query_row("SELECT etag, last_modified, last_checked FROM feeds", [], |r| { - Ok((r.get(0)?, r.get(1)?, r.get(2)?)) - }) - .unwrap() - }; - migrate(&conn).unwrap(); - assert_eq!(row(&conn), (None, None, Some(5)), "re-read on its normal schedule"); - - // Only the once: every later open keeps the validators the next poll stored. - conn.execute_batch("UPDATE feeds SET etag = 'e2'").unwrap(); - migrate(&conn).unwrap(); - assert_eq!(row(&conn).0.as_deref(), Some("e2")); + assert_eq!(db.merge_repeated_enclosures(key).await.unwrap(), (2, vec!["/d/a.mp3".to_string()])); + assert_eq!( + db.i64s_for_test("SELECT id FROM enclosures ORDER BY id").await, + [1, 3, 5, 6, 7], + "a lone ?_=1 and two different files stay" + ); + assert_eq!( + db.strings_for_test("SELECT path FROM enclosures WHERE id = 3 UNION ALL SELECT state FROM enclosures WHERE id = 3") + .await, + ["/d/b.mp3", "done"], + "the only copy moves, not deleted" + ); + assert_eq!(db.merge_repeated_enclosures(key).await.unwrap(), (0, vec![]), "and only once"); } #[tokio::test] @@ -1762,7 +1606,7 @@ mod tests { INSERT INTO enclosures (id, feed_id, guid, url, mime, length, state) VALUES (1,'f','a','u1','audio/mpeg',300,'pending'),(2,'g','b','u2','image/png',10,'pending'), (3,'f','c','u3','video/mp4',2000,'pending');", - ) + ).await .unwrap(); let order = async |col: &str, dir: &str| -> Vec<String> { db.entries_in(1, None, Filter::All, None, 0, 50, &order_sql(col, dir, false)).await @@ -1804,7 +1648,7 @@ mod tests { INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0); INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'f','a','u1','/tmp/a','done');", - ) + ).await .unwrap(); // Nobody has touched it: both others still have it unplayed. @@ -1828,7 +1672,7 @@ mod tests { "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('f','a','One',100),('f','b','Two',200);", - ) + ).await .unwrap(); assert_eq!(db.unread_count(1, "f").await.unwrap(), 2); @@ -1861,69 +1705,15 @@ mod tests { #[tokio::test] async fn schema_is_idempotent_and_summary_handles_unknown_feeds() { let db = Db::memory().await.unwrap(); - // Re-running the schema must not fail: open() does this on every start. - db.conn.lock().unwrap().execute_batch(SCHEMA).unwrap(); + // Creating what is missing again must not fail: open() does it on every start. + create_missing(&db.orm).await.unwrap(); - let sum = db.feed_summary("never-seen").unwrap(); + let sum = db.feed_summary("never-seen").await.unwrap(); assert_eq!(sum.last_checked, None); assert_eq!(sum.entries, 0); assert_eq!(sum.downloaded, 0); } - #[test] - fn an_old_database_loses_its_retired_columns() { - let conn = Connection::open_in_memory().unwrap(); - // As open() has it: a DROP COLUMN on a table that references another is the part worth - // proving, and it has to work with the foreign keys switched on. - conn.pragma_update(None, "foreign_keys", "ON").unwrap(); - conn.execute_batch( - "CREATE TABLE entries (feed_id TEXT NOT NULL, guid TEXT NOT NULL, - first_seen INTEGER NOT NULL, read INTEGER NOT NULL DEFAULT 0, - flagged INTEGER NOT NULL DEFAULT 0, position INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (feed_id, guid)); - CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE COLLATE NOCASE, - pass_hash TEXT, is_admin INTEGER NOT NULL DEFAULT 0, created INTEGER NOT NULL); - CREATE TABLE subscriptions ( - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - feed_id TEXT NOT NULL, created INTEGER NOT NULL, PRIMARY KEY (user_id, feed_id)); - CREATE TABLE sessions (token TEXT PRIMARY KEY, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - created INTEGER NOT NULL, seen INTEGER NOT NULL); - INSERT INTO users VALUES (1, 'ray', NULL, 1, 0); - INSERT INTO subscriptions VALUES (1, 'f', 0); - INSERT INTO sessions VALUES ('t', 1, 0, 0);", - ) - .unwrap(); - // The same order as open(): the schema leaves the old tables alone, migrate() fixes them. - conn.execute_batch(SCHEMA).unwrap(); - migrate(&conn).unwrap(); - let cols = |table: &str| -> Vec<String> { - conn.prepare(&format!("PRAGMA table_info({table})")) - .unwrap() - .query_map([], |r| r.get(1)) - .unwrap() - .collect::<rusqlite::Result<_>>() - .unwrap() - }; - for (table, gone) in [ - ("entries", &["read", "flagged", "position"][..]), - ("subscriptions", &["created"][..]), - ("sessions", &["created"][..]), - ] { - let cols = cols(table); - assert!(!cols.iter().any(|c| gone.contains(&c.as_str())), "{table}: {cols:?}"); - } - // users.created is not retired: it keeps what it held, and last_login joins it. - let users = cols("users"); - assert!(users.iter().any(|c| c == "last_login"), "{users:?}"); - assert_eq!(conn.query_row("SELECT created FROM users", [], |r| r.get::<_, i64>(0)).unwrap(), 0); - // And the rows come through it. - let kept: i64 = conn - .query_row("SELECT count(*) FROM subscriptions JOIN sessions USING (user_id)", [], |r| r.get(0)) - .unwrap(); - assert_eq!(kept, 1); - } - #[tokio::test] async fn a_renamed_account_keeps_everything_but_its_name() { let db = Db::memory().await.unwrap(); @@ -1948,10 +1738,10 @@ mod tests { db.signed_in(id).await.unwrap(); let first = get().await.last_login.unwrap(); // Within the hour, the proxy vouching again writes nothing; after it, it does. - db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 60)).unwrap(); + db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 60)).await.unwrap(); db.signed_in(id).await.unwrap(); assert_eq!(get().await.last_login, Some(first - 60)); - db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 7200)).unwrap(); + db.exec_for_test(&format!("UPDATE users SET last_login = {} WHERE id = {id}", first - 7200)).await.unwrap(); db.signed_in(id).await.unwrap(); assert!(get().await.last_login.unwrap() >= first); } @@ -1961,18 +1751,16 @@ mod tests { // Cutting this along with the dead read columns left the browser suite's admin with an // empty sidebar: it is how a fresh install's first account gets config.toml's feeds. let db = Db::memory().await.unwrap(); - db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',1);") + db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',1);").await .unwrap(); - let subs = || -> i64 { - db.conn.lock().unwrap().query_row("SELECT count(*) FROM subscriptions", [], |r| r.get(0)).unwrap() - }; + let subs = async || db.i64s_for_test("SELECT count(*) FROM subscriptions").await[0]; let catalogue = ["a".to_string(), "b".to_string()]; assert_eq!(db.adopt_catalogue(1, &catalogue).await.unwrap(), 2); - assert_eq!(subs(), 2); + assert_eq!(subs().await, 2); // Once anyone subscribes to anything it never runs again, so an unsubscribe sticks. db.unsubscribe(1, "a").await.unwrap(); assert_eq!(db.adopt_catalogue(1, &catalogue).await.unwrap(), 0); - assert_eq!(subs(), 1); + assert_eq!(subs().await, 1); } #[tokio::test] @@ -2006,7 +1794,7 @@ mod tests { -- Opened, and so read, but 42 of 900 seconds in: still Currently Listening. -- Filtering on read hid exactly these (issue #14). (7,'f','h',1,0,42);", - ) + ).await .unwrap(); for f in [Filter::All, Filter::Unread, Filter::Downloaded, Filter::Flagged, Filter::InProgress] { @@ -2061,7 +1849,7 @@ mod tests { (1,'f','old','u-old','pending'), (2,'f','mid','u-mid','pending'), (3,'f','new','u-new','pending');", - ) + ).await .unwrap(); let got: Vec<String> = db.pending("f", 2).await.unwrap().into_iter().map(|p| p.url).collect(); assert_eq!(got, vec!["u-new", "u-mid"], "newest first, oldest left for later"); @@ -2076,16 +1864,13 @@ mod tests { (2,'f','b','u2','pending',NULL), (3,'f','c','u3','downloading','/tmp/already-here'), (4,'f','d','u4','done','/tmp/x');", - ) + ).await .unwrap(); assert_eq!(db.requeue_interrupted().await.unwrap(), 1, "only the in-flight, fileless one"); - let conn = db.conn.lock().unwrap(); - let state = |id: i64| -> String { - conn.query_row("SELECT state FROM enclosures WHERE id = ?1", [id], |r| r.get(0)).unwrap() - }; - assert_eq!(state(1), "pending"); - assert_eq!(state(3), "downloading", "it has a file; leave it alone"); - assert_eq!(state(4), "done"); + let state = async |id: i64| db.strings_for_test(&format!("SELECT state FROM enclosures WHERE id = {id}")).await; + assert_eq!(state(1).await, ["pending"]); + assert_eq!(state(3).await, ["downloading"], "it has a file; leave it alone"); + assert_eq!(state(4).await, ["done"]); } #[tokio::test] @@ -2099,29 +1884,25 @@ mod tests { (3,'creator','c','u3','skipped',NULL,'explicit'), (4,'other','d','u4','skipped',NULL,'torrents disabled'); INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',1);", - ) + ).await .unwrap(); - db.adopt("creator", "show", &[("a", "u1"), ("b", "u2"), ("d", "u4")]).unwrap(); - { - let conn = db.conn.lock().unwrap(); - let owner = |id: i64| -> String { - conn.query_row("SELECT feed_id FROM enclosures WHERE id = ?1", [id], |r| r.get(0)).unwrap() - }; - assert_eq!(owner(1), "show", "a downloaded file moves with its item"); - assert_eq!(owner(2), "show"); - assert_eq!(owner(3), "creator", "this show does not list it"); - assert_eq!(owner(4), "other", "only the parent's are taken"); - let read: String = conn - .query_row("SELECT feed_id FROM entry_state WHERE user_id = 1 AND guid = 'a'", [], |r| r.get(0)) - .unwrap(); - assert_eq!(read, "show", "what you had read stays read"); - } + db.adopt("creator", "show", &[("a", "u1"), ("b", "u2"), ("d", "u4")]).await.unwrap(); + let owner = async |id: i64| db.strings_for_test(&format!("SELECT feed_id FROM enclosures WHERE id = {id}")).await; + assert_eq!(owner(1).await, ["show"], "a downloaded file moves with its item"); + assert_eq!(owner(2).await, ["show"]); + assert_eq!(owner(3).await, ["creator"], "this show does not list it"); + assert_eq!(owner(4).await, ["other"], "only the parent's are taken"); + assert_eq!( + db.strings_for_test("SELECT feed_id FROM entry_state WHERE user_id = 1 AND guid = 'a'").await, + ["show"], + "what you had read stays read" + ); // Only a filter's verdict can be overturned by a change of settings. - let skipped = db.skipped_by_filter("show").unwrap(); + let skipped = db.skipped_by_filter("show").await.unwrap(); assert_eq!(skipped.get("u2").map(String::as_str), Some("explicit")); assert_eq!(skipped.len(), 1); - assert!(db.skipped_by_filter("other").unwrap().is_empty(), "torrents disabled is not a filter"); + assert!(db.skipped_by_filter("other").await.unwrap().is_empty(), "torrents disabled is not a filter"); } #[tokio::test] @@ -2131,7 +1912,7 @@ mod tests { "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); INSERT INTO subscriptions (user_id, feed_id, allow_explicit) VALUES (1,'group',1),(1,'show',NULL),(2,'group',1),(2,'show',0);", - ) + ).await .unwrap(); let explicit = async |group| -> Vec<Option<bool>> { let mut v: Vec<_> = @@ -2146,19 +1927,19 @@ mod tests { #[tokio::test] async fn error_since_marks_the_start_of_a_run_of_failures_and_clears_on_success() { let db = Db::memory().await.unwrap(); - db.set_feed_error("f", "http://x", "HTTP 404").unwrap(); + db.set_feed_error("f", "http://x", "HTTP 404").await.unwrap(); // Backdate it, as if this feed had already been failing a while, so a second // failure landing "now" is distinguishable from the first. - db.exec_for_test("UPDATE feeds SET error_since = error_since - 3600 WHERE id = 'f'").unwrap(); - let first = db.feed_summary("f").unwrap().error_since.unwrap(); + db.exec_for_test("UPDATE feeds SET error_since = error_since - 3600 WHERE id = 'f'").await.unwrap(); + let first = db.feed_summary("f").await.unwrap().error_since.unwrap(); // macmanx: failed once, read fine an hour later. A second failure must not push // error_since forward -- the UI decides "failing for a day" from the first one. - db.set_feed_error("f", "http://x", "HTTP 404").unwrap(); - assert_eq!(db.feed_summary("f").unwrap().error_since, Some(first)); + db.set_feed_error("f", "http://x", "HTTP 404").await.unwrap(); + assert_eq!(db.feed_summary("f").await.unwrap().error_since, Some(first)); - db.touch_feed("f", "http://x").unwrap(); - let after = db.feed_summary("f").unwrap(); + db.touch_feed("f", "http://x").await.unwrap(); + let after = db.feed_summary("f").await.unwrap(); assert_eq!(after.last_error, None); assert_eq!(after.error_since, None, "a clean check ends the run of failures"); } @@ -2181,9 +1962,8 @@ mod tests { #[tokio::test] async fn enclosure_url_is_the_dedupe_key() { let db = Db::memory().await.unwrap(); - let conn = db.conn.lock().unwrap(); let insert = "INSERT INTO enclosures (feed_id, guid, url, state) VALUES ('f', 'g', 'http://x/a.mp3', 'pending')"; - conn.execute(insert, []).unwrap(); - assert!(conn.execute(insert, []).is_err(), "duplicate url must be rejected"); + db.exec_for_test(insert).await.unwrap(); + assert!(db.exec_for_test(insert).await.is_err(), "duplicate url must be rejected"); } } diff --git a/src/entity.rs b/src/entity.rs index f8a05cb..a8ff7bb 100644 --- a/src/entity.rs +++ b/src/entity.rs @@ -1,6 +1,6 @@ //! The database's tables as SeaORM entities: the one description of the schema, from which -//! `Db::open` creates what a database is missing, on SQLite or Postgres alike (see `db::sync`). -//! What each column means is in the comments on `db::SCHEMA`, the SQLite schema these match. +//! `Db::open` creates what a database is missing, on SQLite or Postgres alike (see +//! `db::create_missing`). Times are Unix seconds. pub mod feeds { use sea_orm::entity::prelude::*; @@ -16,6 +16,7 @@ pub mod feeds { pub title: Option<String>, #[sea_orm(column_type = "Text", nullable)] pub image: Option<String>, + /// The channel's first <itunes:category>, for the Directory. #[sea_orm(column_type = "Text", nullable)] pub category: Option<String>, #[sea_orm(column_type = "Text", nullable)] @@ -26,11 +27,19 @@ pub mod feeds { pub ttl_mins: Option<i64>, #[sea_orm(column_type = "Text", nullable)] pub last_error: Option<String>, + /// When the current run of failures began; NULL while the feed is healthy. Kept through + /// repeated failures so the UI can tell a blip (macmanx: failed once, fine an hour + /// later) from a feed that has been down for a day. pub error_since: Option<i64>, + /// Came from a subscribed OPML that no longer lists it, but has downloads, so kept. #[sea_orm(default_value = false)] pub orphaned: bool, + /// The OPML subscription this feed came from. #[sea_orm(column_type = "Text", nullable)] pub group_id: Option<String>, + /// Derived from an OPML and not written to config.toml. Writing 80-odd generated entries + /// into a hand-edited file made it unreadable; the OPML is the source of truth, so they + /// are re-derived instead. Customising one promotes it to config. #[sea_orm(default_value = false)] pub managed: bool, } @@ -84,7 +93,8 @@ pub mod enclosures { pub feed_id: String, #[sea_orm(column_type = "Text")] pub guid: String, - /// The dedupe key: one file serves every subscriber. + /// The dedupe key, and the reason one file serves every subscriber. A reaped file keeps + /// its row with path NULL and state 'reaped', so a purged episode is never fetched again. #[sea_orm(unique, column_type = "Text")] pub url: String, #[sea_orm(column_type = "Text", nullable)] @@ -115,16 +125,20 @@ pub mod users { pub struct Model { #[sea_orm(primary_key)] pub id: i64, - /// Unique without regard to case: `db::sync` adds the index on lower(name), which + /// Unique without regard to case: `db::create_missing` adds the index on lower(name), which /// works the same on both databases where SQLite's COLLATE NOCASE does not. #[sea_orm(column_type = "Text")] pub name: String, + /// NULL for someone who only ever arrives through the proxy: there is no password to + /// check, and leaving it empty is not the same as leaving it unset. #[sea_orm(column_type = "Text", nullable)] pub pass_hash: Option<String>, #[sea_orm(default_value = false)] pub is_admin: bool, + /// For whoever maintains the server. NULL where it is not known. pub created: Option<i64>, pub last_login: Option<i64>, + /// The theme chosen in Settings, and light, dark or auto. NULL until one is chosen. #[sea_orm(column_type = "Text", nullable)] pub theme: Option<String>, #[sea_orm(column_type = "Text", nullable)] @@ -161,6 +175,8 @@ macro_rules! owned_by_user { }; } +/// What one person wants from a feed. The feed, its items and its files are shared; this is the +/// part that is not. NULL in a column means: follow the feed's own setting. pub mod subscriptions { use sea_orm::entity::prelude::*; @@ -177,6 +193,7 @@ pub mod subscriptions { pub auto_download: Option<bool>, pub allow_explicit: Option<bool>, pub max_new_per_check: Option<i64>, + /// Pinned to the top of this person's feed list, a feed inside a folder included. #[sea_orm(default_value = false)] pub pinned: bool, } @@ -184,6 +201,8 @@ pub mod subscriptions { owned_by_user!(); } +/// Read, kept and how far in. One row per person per item, created on first touch; an item +/// nobody has touched has no row at all, which is what unread means. pub mod entry_state { use sea_orm::entity::prelude::*; @@ -202,6 +221,7 @@ pub mod entry_state { pub flagged: bool, #[sea_orm(default_value = 0)] pub position: i64, + /// The length this person's player measured, beside the position it is measured against. pub duration: Option<i64>, } diff --git a/src/main.rs b/src/main.rs index 6a227f0..595c3a4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -220,7 +220,7 @@ async fn main() -> Result<()> { }); match cli.command { - Command::List => list(&ctx, &config_path), + Command::List => list(&ctx, &config_path).await, Command::Daemon { web } => daemon(ctx, config_path, web, events).await, Command::Add { url, folder, keywords } => { add(&ctx, &config_path, &url, folder, keywords).await @@ -228,7 +228,7 @@ async fn main() -> Result<()> { Command::Rm { feed } => rm(&ctx, &config_path, &feed).await, Command::User { cmd } => user_cmd(&ctx, cmd).await, Command::Import { file } => import(&ctx, &config_path, &file).await, - Command::Export { file } => export(&ctx, &file), + Command::Export { file } => export(&ctx, &file).await, _ => run(&ctx, wire_cmd.expect("only List and Daemon have no wire form")).await, } } @@ -351,7 +351,7 @@ async fn run(ctx: &Arc<Ctx>, cmd: Cmd) -> Result<()> { async fn status(ctx: &Ctx) -> Event { match ctx.db.counts().await { Ok((pending, downloaded)) => { - let feeds = subscriptions(ctx).map(|s| s.len()).unwrap_or(0); + let feeds = subscriptions(ctx).await.map(|s| s.len()).unwrap_or(0); Event::Status { feeds, pending, downloaded } } Err(e) => Event::Error { msg: format!("{e:#}") }, @@ -401,7 +401,7 @@ async fn daemon( // Before the parser knew WordPress's numbered player URLs, a file it listed twice was // downloaded twice. The repeats fold into the first, and their spare copies are deleted. - match ctx.db.merge_repeated_enclosures(feed::same_file_key) { + match ctx.db.merge_repeated_enclosures(feed::same_file_key).await { Ok((0, _)) => {} Ok((n, spare)) => { for path in &spare { @@ -431,7 +431,7 @@ async fn daemon( let mut ticker = tokio::time::interval(std::time::Duration::from_secs(60)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); tracing::info!( - feeds = subscriptions(&ctx).map(|s| s.len()).unwrap_or(0), + feeds = subscriptions(&ctx).await.map(|s| s.len()).unwrap_or(0), "daemon started" ); @@ -575,7 +575,7 @@ async fn add( let mut cfg = (*ctx.cfg()).clone(); let url = &feed::expand_input(url); // Includes feeds derived from an OPML, or the same show could be added twice. - if let Some(existing) = subscriptions(ctx)?.iter().find(|s| feed::same_feed(&s.cfg.url, url)) { + if let Some(existing) = subscriptions(ctx).await?.iter().find(|s| feed::same_feed(&s.cfg.url, url)) { anyhow::bail!("already subscribed as {:?}", existing.id); } let id = add_one(ctx, &mut cfg, url, folder, keywords).await?; @@ -627,13 +627,13 @@ pub async fn add_one( // Slugs must be unique across derived feeds too, or a new feed can collide with one // an OPML already introduced. - let mut taken: std::collections::BTreeMap<String, config::Feed> = subscriptions(ctx)? + let mut taken: std::collections::BTreeMap<String, config::Feed> = subscriptions(ctx).await? .into_iter() .map(|s| (s.id, s.cfg)) .collect(); // A removed feed keeps its rows, so its id is only free again for the same feed: re-adding // it gets its history back, and a different feed does not inherit someone else's. - for (id, other) in ctx.db.feed_urls()? { + for (id, other) in ctx.db.feed_urls().await? { if !feed::same_feed(&other, url) { taken.entry(id).or_insert_with(|| probe.clone()); } @@ -656,7 +656,7 @@ async fn rm(ctx: &Ctx, config_path: &std::path::Path, feed: &str) -> Result<()> if cfg.feeds.remove(feed).is_none() { // Derived from an OPML: drop it here, though the subscription will list it again // on the next read unless the OPML itself goes. - ctx.db.drop_managed(feed)?; + ctx.db.drop_managed(feed).await?; println!("removed {feed}; it came from an OPML subscription and may return on the next read"); return Ok(()); } @@ -703,7 +703,7 @@ pub async fn subscribe_opml( let mut found = vec![]; collect_outlines(&doc.body.outlines, &mut found); - let known = subscriptions(ctx)?; + let known = subscriptions(ctx).await?; let mut cfg = (*ctx.cfg()).clone(); let mut ids = vec![]; let mut grew = false; @@ -770,7 +770,7 @@ pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, Strin } } -fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> { +async fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> { let mut doc = opml::OPML::default(); doc.head = Some(opml::Head { title: Some("ipx subscriptions".into()), @@ -779,7 +779,7 @@ fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> { for (id, feed) in &ctx.cfg().feeds { let title = ctx .db - .feed_summary(id) + .feed_summary(id).await .ok() .and_then(|s| s.title) .unwrap_or_else(|| id.clone()); @@ -791,14 +791,14 @@ fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> { Ok(()) } -fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> { +async fn list(ctx: &Ctx, config_path: &std::path::Path) -> Result<()> { let cfg = ctx.cfg(); if cfg.feeds.is_empty() { println!("No feeds configured in {}", config_path.display()); return Ok(()); } for (id, feed) in &cfg.feeds { - let s = ctx.db.feed_summary(id)?; + let s = ctx.db.feed_summary(id).await?; println!("{id} {}", s.title.as_deref().unwrap_or("-")); println!(" url {}", feed.url); println!(" last checked {}", ago(s.last_checked)); @@ -832,7 +832,7 @@ async fn reap(ctx: &Ctx, dry_run: bool, standalone: bool) -> Result<()> { async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> { let cfg = ctx.cfg(); - let subs = subscriptions(ctx)?; + let subs = subscriptions(ctx).await?; if let Some(id) = only && !subs.iter().any(|s| s.id == id) { @@ -843,7 +843,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> { let mut fresh: Vec<String> = vec![]; for sub in subs.iter().filter(|s| only.is_none_or(|o| o == s.id)) { let (id, feed_cfg) = (&sub.id, &sub.cfg); - let state = ctx.db.http_state(id)?; + let state = ctx.db.http_state(id).await?; if !force && let Some(last) = state.last_checked { let due = last + due_after(&cfg, feed_cfg, state.ttl_mins) as i64; @@ -890,20 +890,20 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> { // One bad feed must not end the scan. let msg = format!("{e:#}"); ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() }); - ctx.db.set_feed_error(id, &feed_cfg.url, &msg)?; + ctx.db.set_feed_error(id, &feed_cfg.url, &msg).await?; } } } // Feeds a subscribed OPML just introduced: scan them now, in this run. if !fresh.is_empty() { - let subs = subscriptions(ctx)?; + let subs = subscriptions(ctx).await?; for id in &fresh { let Some(feed_cfg) = subs.iter().find(|s| &s.id == id).map(|s| &s.cfg) else { continue; }; scanned += 1; ctx.out.emit(Event::FeedStart { feed: id.clone() }); - let state = ctx.db.http_state(id)?; + let state = ctx.db.http_state(id).await?; match scan_one(ctx, id, feed_cfg, &state).await { Ok(Outcome::Feed(s)) => ctx.out.emit(Event::FeedDone { feed: id.clone(), @@ -916,7 +916,7 @@ async fn fetch(ctx: &Arc<Ctx>, only: Option<&str>, force: bool) -> Result<()> { Err(e) => { let msg = format!("{e:#}"); ctx.out.emit(Event::FeedError { feed: id.clone(), msg: msg.clone() }); - ctx.db.set_feed_error(id, &feed_cfg.url, &msg)?; + ctx.db.set_feed_error(id, &feed_cfg.url, &msg).await?; } } } @@ -938,7 +938,7 @@ pub struct Sub { /// /// A derived feed borrows its parent's settings wholesale. That is why it needs no config /// entry -- there is nothing to store but its URL and where it came from. -pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> { +pub async fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> { let cfg = ctx.cfg(); let mut out: Vec<Sub> = cfg .feeds @@ -946,7 +946,7 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> { .map(|(id, f)| Sub { id: id.clone(), cfg: f.clone(), managed: false }) .collect(); - for m in ctx.db.managed_feeds()? { + for m in ctx.db.managed_feeds().await? { if cfg.feeds.contains_key(&m.id) { continue; // promoted to config at some point; that entry wins } @@ -957,10 +957,10 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> { // skip them here regardless so a row that slips through is never scanned. continue; } - let base = parent - .and_then(|p| p.folder.clone()) - .or_else(|| ctx.db.feed_summary(&m.group_id).ok().and_then(|s| s.title)) - .unwrap_or_else(|| m.group_id.clone()); + let base = match parent.and_then(|p| p.folder.clone()) { + Some(folder) => folder, + None => ctx.db.feed_summary(&m.group_id).await.ok().and_then(|s| s.title).unwrap_or_else(|| m.group_id.clone()), + }; let title = m.title.clone().unwrap_or_else(|| m.id.clone()); out.push(Sub { id: m.id.clone(), @@ -994,15 +994,15 @@ pub fn subscriptions(ctx: &Ctx) -> Result<Vec<Sub>> { /// derived any more, so it is only unmanaged. pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> { let cfg = ctx.cfg(); - for m in ctx.db.managed_feeds()?.into_iter().filter(|m| m.group_id == parent_id) { + for m in ctx.db.managed_feeds().await?.into_iter().filter(|m| m.group_id == parent_id) { if cfg.feeds.contains_key(&m.id) { // Scanned from its config entry and still read. Dropped as derived, its stored // entries would go with it: davewiner's 11 were promoted without being unmanaged. - ctx.db.unmanage(&m.id)?; + ctx.db.unmanage(&m.id).await?; } else if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 { - ctx.db.set_orphaned(&m.id, true)?; + ctx.db.set_orphaned(&m.id, true).await?; } else { - ctx.db.drop_managed(&m.id)?; + ctx.db.drop_managed(&m.id).await?; } } Ok(()) @@ -1014,7 +1014,7 @@ pub async fn retire_group(ctx: &Ctx, parent_id: &str) -> Result<()> { /// most of the ones stored. async fn retire_stranded(ctx: &Ctx) -> Result<usize> { let cfg = ctx.cfg(); - let before = ctx.db.managed_feeds()?; + let before = ctx.db.managed_feeds().await?; let stranded: std::collections::BTreeSet<&str> = before .iter() .map(|m| m.group_id.as_str()) @@ -1023,7 +1023,7 @@ async fn retire_stranded(ctx: &Ctx) -> Result<usize> { for group in stranded { retire_group(ctx, group).await?; } - Ok(before.len() - ctx.db.managed_feeds()?.len()) + Ok(before.len() - ctx.db.managed_feeds().await?.len()) } /// Seconds to wait before re-checking a feed. @@ -1068,19 +1068,19 @@ async fn scan_one( if feed::is_patreon_creator(&feed_cfg.url) { match feed::patreon_shows(&ctx.client, &feed_cfg.url).await { Ok((name, shows)) if shows.len() > 1 => { - ctx.db.touch_feed(id, &feed_cfg.url)?; + ctx.db.touch_feed(id, &feed_cfg.url).await?; if let Some(name) = name { - ctx.db.set_title(id, &name)?; + ctx.db.set_title(id, &name).await?; } // Read as one feed before it was split, it listed every show's items in one // heap. The items go; its files and read state move to each show as the show // lists them (`Db::adopt`), so no show comes up empty for want of a URL. - ctx.db.clear_entries(id)?; + ctx.db.clear_entries(id).await?; return sync_group(ctx, id, feed_cfg, &shows).await; } Ok(_) => {} // One show: the creator's feed is that show. // Already split: keep the shows it has rather than read the creator as one heap. - Err(e) if ctx.db.managed_feeds()?.iter().any(|m| m.group_id == id) => return Err(e), + Err(e) if ctx.db.managed_feeds().await?.iter().any(|m| m.group_id == id) => return Err(e), Err(e) => tracing::warn!( feed = id, error = %format!("{e:#}"), @@ -1101,29 +1101,29 @@ async fn scan_one( // from backup, a manual edit, a cleanup that removed entries. Believe the database over // the validator: drop it and ask again, or the feed stays empty until the publisher // happens to change something. - if matches!(fetched, feed::Fetched::NotModified) && ctx.db.feed_summary(id)?.entries == 0 { + if matches!(fetched, feed::Fetched::NotModified) && ctx.db.feed_summary(id).await?.entries == 0 { tracing::info!(feed = id, "not modified, but nothing stored; refetching without the validator"); - ctx.db.clear_validators(id)?; + ctx.db.clear_validators(id).await?; fetched = feed::fetch(&ctx.client, feed_cfg, None, None).await?; } let (bytes, etag, last_modified) = match fetched { feed::Fetched::NotModified => { - ctx.db.touch_feed(id, &feed_cfg.url)?; + ctx.db.touch_feed(id, &feed_cfg.url).await?; return Ok(Outcome::NotModified); } feed::Fetched::Body { bytes, etag, last_modified } => (bytes, etag, last_modified), }; if bytes.iter().all(u8::is_ascii_whitespace) { - ctx.db.touch_feed(id, &feed_cfg.url)?; + ctx.db.touch_feed(id, &feed_cfg.url).await?; return Ok(Outcome::Empty); } // A subscribed OPML is a list of feeds, not a feed. The original matched on a ".opml" // URL; sniffing the body also catches one served from a URL without that extension. if feed::is_opml(&bytes) { - ctx.db.touch_feed(id, &feed_cfg.url)?; + ctx.db.touch_feed(id, &feed_cfg.url).await?; return sync_opml(ctx, id, feed_cfg, &bytes).await; } @@ -1137,7 +1137,7 @@ async fn scan_one( parsed.ttl_mins, parsed.image.as_deref(), parsed.category.as_deref(), - )?; + ).await?; let policy = policy_for(ctx, id, feed_cfg).await?; if let Some(parent) = &feed_cfg.group { @@ -1146,16 +1146,16 @@ async fn scan_one( .iter() .flat_map(|e| e.enclosures.iter().map(move |x| (e.guid.as_str(), x.url.as_str()))) .collect(); - ctx.db.adopt(parent, id, &listed)?; + ctx.db.adopt(parent, id, &listed).await?; } // Verdicts are recorded in `state`, so the download queue below is just "everything still // pending". A filter's verdict is looked at again on every scan, though: made once, at // discovery, it outlived the setting behind it, and allowing explicit items afterwards // changed nothing however often the feed was scanned. - let skipped = ctx.db.skipped_by_filter(id)?; + let skipped = ctx.db.skipped_by_filter(id).await?; let mut scan = Scan::default(); for entry in &parsed.entries { - if ctx.db.record_entry(id, entry)? { + if ctx.db.record_entry(id, entry).await? { scan.new_entries += 1; } for enc in &entry.enclosures { @@ -1263,7 +1263,7 @@ async fn sync_opml( ) -> Result<Outcome> { let listed = feed::parse_opml(bytes)?; if let Some(title) = feed::opml_title(bytes) { - ctx.db.set_title(parent_id, &title)?; + ctx.db.set_title(parent_id, &title).await?; } sync_group(ctx, parent_id, parent, &listed).await } @@ -1282,13 +1282,13 @@ async fn sync_group( listed: &[(String, String)], ) -> Result<Outcome> { let cfg = ctx.cfg(); - let existing = ctx.db.managed_feeds()?; + let existing = ctx.db.managed_feeds().await?; let mut added = vec![]; for (title, url) in listed { // Already known, whether derived or promoted into the config. if let Some(m) = existing.iter().find(|m| &m.url == url) { - ctx.db.upsert_managed(&m.id, url, title, parent_id)?; + ctx.db.upsert_managed(&m.id, url, title, parent_id).await?; continue; } // A Patreon show you added by hand may be spelled differently from the one listed. @@ -1296,7 +1296,7 @@ async fn sync_group( continue; } // A removed feed keeps its rows, so its id is only free again for the same feed. - let known = ctx.db.feed_urls()?; + let known = ctx.db.feed_urls().await?; let taken: std::collections::BTreeMap<String, config::Feed> = cfg .feeds .keys() @@ -1306,7 +1306,7 @@ async fn sync_group( .map(|id| (id.clone(), parent.clone())) .collect(); let id = config::unique_slug(title, &taken); - ctx.db.upsert_managed(&id, url, title, parent_id)?; + ctx.db.upsert_managed(&id, url, title, parent_id).await?; added.push(id); } @@ -1314,7 +1314,7 @@ async fn sync_group( // subscription means. Their own feeds are untouched. for id in ctx .db - .managed_feeds()? + .managed_feeds().await? .iter() .filter(|m| m.group_id == parent_id) .map(|m| m.id.clone()) @@ -1336,11 +1336,11 @@ async fn sync_group( } if ctx.db.downloaded_count(&m.id).await.unwrap_or(1) > 0 { // Never orphan a downloaded file: keep the feed and say why in the UI. - ctx.db.set_orphaned(&m.id, true)?; + ctx.db.set_orphaned(&m.id, true).await?; kept += 1; tracing::info!(feed = %m.id, "dropped from the OPML but has downloads; keeping it"); } else { - ctx.db.drop_managed(&m.id)?; + ctx.db.drop_managed(&m.id).await?; removed += 1; tracing::info!(feed = %m.id, "dropped from the OPML with nothing downloaded; removed"); } @@ -1546,7 +1546,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> { } // Must look through the derived feeds too: anything inside an OPML subscription has // no config entry, so a config-only lookup called every one of them "unsubscribed". - let subs = subscriptions(ctx)?; + let subs = subscriptions(ctx).await?; let feed_cfg = subs .iter() .find(|s| s.id == enc.feed_id) @@ -1554,7 +1554,7 @@ async fn download_one(ctx: &Arc<Ctx>, id: i64) -> Result<()> { .ok_or_else(|| anyhow::anyhow!("enclosure {id} belongs to unsubscribed feed {:?}", enc.feed_id))?; let feed_cfg = &feed_cfg; - let title = ctx.db.feed_summary(&enc.feed_id)?.title; + let title = ctx.db.feed_summary(&enc.feed_id).await?.title; let folder = download::folder_for(&cfg, &enc.feed_id, feed_cfg, title.as_deref()); let dest_dir = cfg.general.download_dir.join(&folder); @@ -1743,9 +1743,9 @@ mod tests { // davewiner: the OPML subscription left config.toml, but its 922 derived rows // stayed in the database and kept being scanned under the no-parent fallback. let ctx = test_ctx(config::Config::default()).await; - ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").unwrap(); + ctx.db.upsert_managed("child", "http://x/child.xml", "Child", "gone-opml").await.unwrap(); assert!( - subscriptions(&ctx).unwrap().iter().all(|s| s.id != "child"), + subscriptions(&ctx).await.unwrap().iter().all(|s| s.id != "child"), "a derived feed whose parent is gone from config must not be scanned" ); } @@ -1753,18 +1753,18 @@ mod tests { #[tokio::test] async fn retiring_a_group_drops_what_was_never_downloaded_and_orphans_the_rest() { let ctx = test_ctx(config::Config::default()).await; - ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").unwrap(); - ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").unwrap(); + ctx.db.upsert_managed("empty", "http://x/empty.xml", "Empty", "parent").await.unwrap(); + ctx.db.upsert_managed("has-file", "http://x/has-file.xml", "Has File", "parent").await.unwrap(); let enc = feed::Enclosure { url: "http://x/ep.mp3".into(), mime: None, length: None }; ctx.db.record_enclosure("has-file", "g1", &enc).await.unwrap(); ctx.db.mark_downloaded(&enc.url, std::path::Path::new("/downloads/ep.mp3"), 1).await.unwrap(); retire_group(&ctx, "parent").await.unwrap(); - let managed = ctx.db.managed_feeds().unwrap(); + let managed = ctx.db.managed_feeds().await.unwrap(); assert!(!managed.iter().any(|m| m.id == "empty"), "nothing downloaded, so it is forgotten"); assert!(managed.iter().any(|m| m.id == "has-file"), "has a file on disk, so it is kept"); - assert!(ctx.db.feed_summary("has-file").unwrap().orphaned, "and flagged as orphaned"); + assert!(ctx.db.feed_summary("has-file").await.unwrap().orphaned, "and flagged as orphaned"); } #[tokio::test] @@ -1775,15 +1775,15 @@ mod tests { cfg.feeds.insert("promoted".into(), feed()); cfg.feeds.insert("live-opml".into(), feed()); let ctx = test_ctx(cfg).await; - ctx.db.upsert_managed("promoted", "http://x/p.xml", "Promoted", "gone-opml").unwrap(); - ctx.db.upsert_managed("empty", "http://x/e.xml", "Empty", "gone-opml").unwrap(); - ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").unwrap(); - ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).unwrap(); + ctx.db.upsert_managed("promoted", "http://x/p.xml", "Promoted", "gone-opml").await.unwrap(); + ctx.db.upsert_managed("empty", "http://x/e.xml", "Empty", "gone-opml").await.unwrap(); + ctx.db.upsert_managed("listed", "http://x/l.xml", "Listed", "live-opml").await.unwrap(); + ctx.db.record_entry("promoted", &feed::Entry { guid: "g1".into(), ..Default::default() }).await.unwrap(); assert_eq!(retire_stranded(&ctx).await.unwrap(), 2, "empty dropped, promoted unmanaged"); - let managed: Vec<String> = ctx.db.managed_feeds().unwrap().into_iter().map(|m| m.id).collect(); + let managed: Vec<String> = ctx.db.managed_feeds().await.unwrap().into_iter().map(|m| m.id).collect(); assert_eq!(managed, ["listed"], "a group still in config is left alone"); - assert_eq!(ctx.db.feed_summary("promoted").unwrap().entries, 1, "its entries survive"); + assert_eq!(ctx.db.feed_summary("promoted").await.unwrap().entries, 1, "its entries survive"); } } diff --git a/src/retention.rs b/src/retention.rs index a6e81e4..e58f2eb 100644 --- a/src/retention.rs +++ b/src/retention.rs @@ -66,7 +66,7 @@ pub async fn run(cfg: &Config, db: &Db, dry_run: bool) -> Result<Report> { report.bytes_freed += remove(db, c, dry_run).await?; } if !dry_run { - report.entries_pruned = db.prune_entries(cutoff)?; + report.entries_pruned = db.prune_entries(cutoff).await?; } } @@ -173,7 +173,7 @@ mod tests { (2, 'f', 'half', 'u2', '/tmp/half', 10, 'done', 20), (3, 'f', 'unread', 'u3', '/tmp/unread', 10, 'done', 30), (4, 'f', 'read', 'u4', '/tmp/read', 10, 'done', 40);", - ) + ).await .unwrap(); let got: Vec<i64> = db.reap_candidates().await.unwrap().iter().map(|c| c.id).collect(); @@ -198,9 +198,9 @@ mod tests { ('f', 'recent', 900); INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1, 'f', 'has-file', 'u1', '/tmp/x', 'done');", - ) + ).await .unwrap(); - assert_eq!(db.prune_entries(500).unwrap(), 1, "only the old, fileless, unflagged one"); + assert_eq!(db.prune_entries(500).await.unwrap(), 1, "only the old, fileless, unflagged one"); } } diff --git a/src/web.rs b/src/web.rs index 1c897d5..f40ca22 100644 --- a/src/web.rs +++ b/src/web.rs @@ -672,7 +672,7 @@ async fn feeds( let cfg = state.ctx.cfg(); // Config entries plus the feeds derived from OPML subscriptions -- the catalogue. // What comes back is only the part of it this person subscribes to. - let subs = crate::subscriptions(&state.ctx)?; + let subs = crate::subscriptions(&state.ctx).await?; let mine: std::collections::HashMap<String, crate::db::Sub> = state .ctx .db @@ -689,8 +689,8 @@ async fn feeds( // the same fallback the scanner uses (`Db::subscribers`). let up = feed.group.as_deref().and_then(|g| mine.get(g)); let Some(mine) = mine.get(id) else { continue }; - let s = state.ctx.db.feed_summary(id)?; - let st = state.ctx.db.http_state(id)?; + let s = state.ctx.db.feed_summary(id).await?; + let st = state.ctx.db.http_state(id).await?; out.push(FeedRow { id: id.clone(), url: feed.url.clone(), @@ -806,7 +806,7 @@ async fn popular(state: &WebState, user_id: i64) -> Result<Vec<PopularRow>> { db.subscriptions_for(user_id).await?.into_iter().map(|s| s.feed_id).collect(); let counts = db.subscriber_counts().await?; let media = db.media_feeds().await?; - let catalogue = crate::subscriptions(&state.ctx)?; + let catalogue = crate::subscriptions(&state.ctx).await?; let by_id: std::collections::HashMap<&str, &crate::config::Feed> = catalogue.iter().map(|s| (s.id.as_str(), &s.cfg)).collect(); let is_folder: std::collections::HashSet<&str> = @@ -823,7 +823,7 @@ async fn popular(state: &WebState, user_id: i64) -> Result<Vec<PopularRow>> { { continue; } - let sum = db.feed_summary(&s.id)?; + let sum = db.feed_summary(&s.id).await?; let subscribed = mine.contains(&s.id); out.push(PopularRow { id: s.id.clone(), @@ -1219,7 +1219,7 @@ async fn add_feed( let url = crate::feed::expand_input(&body.url); // Someone else may already have it. Then adding costs nothing: no second fetch, no // second copy on disk, just another name against the same feed. - if let Some(existing) = crate::subscriptions(&state.ctx)? + if let Some(existing) = crate::subscriptions(&state.ctx).await? .into_iter() .find(|s| crate::feed::same_feed(&s.cfg.url, &url)) { @@ -1341,13 +1341,13 @@ async fn patch_feed( // Derived feeds have no config entry. Editing one is the moment it earns a real // entry: promote it, so the config holds your decisions and nothing else. if !cfg.feeds.contains_key(&id) { - let subs = crate::subscriptions(&state.ctx)?; + let subs = crate::subscriptions(&state.ctx).await?; let found = subs .iter() .find(|s| s.id == id) .ok_or_else(|| ApiError::bad_request(format!("no feed with id {id:?}")))?; cfg.feeds.insert(id.clone(), found.cfg.clone()); - state.ctx.db.unmanage(&id)?; + state.ctx.db.unmanage(&id).await?; } let checked = match &body.url { @@ -1388,7 +1388,7 @@ async fn patch_feed( if url_changed { // Refreshing a rotated auth token is the common case; entries and download history // are keyed by feed id, so they survive the change. - state.ctx.db.clear_validators(&id)?; + state.ctx.db.clear_validators(&id).await?; } Ok(StatusCode::NO_CONTENT) } @@ -1401,7 +1401,7 @@ async fn remove_feed( // Unsubscribing is personal: it takes the feed off your list and leaves everyone // else's alone. state.ctx.db.unsubscribe(user.id, &id).await?; - for child in crate::subscriptions(&state.ctx)? + for child in crate::subscriptions(&state.ctx).await? .iter() .filter(|s| s.cfg.group.as_deref() == Some(id.as_str())) { @@ -1417,7 +1417,7 @@ async fn remove_feed( if cfg.feeds.remove(&id).is_none() { // A derived feed: forget it here, though the OPML will list it again on the next // read unless you unsubscribe from the OPML itself. - state.ctx.db.drop_managed(&id)?; + state.ctx.db.drop_managed(&id).await?; return Ok(StatusCode::NO_CONTENT); } cfg.save(&state.config_path)?; @@ -1610,7 +1610,7 @@ async fn read_all( // A subscription's own row has no entries, so marking it read means everything under it. let mut ids = vec![id.clone()]; ids.extend( - crate::subscriptions(&state.ctx)? + crate::subscriptions(&state.ctx).await? .into_iter() .filter(|s| s.cfg.group.as_deref() == Some(id.as_str())) .map(|s| s.id), @@ -1676,7 +1676,7 @@ async fn export_opml( }), ..Default::default() }; - for s in crate::subscriptions(&state.ctx)? { + for s in crate::subscriptions(&state.ctx).await? { // A feed from an OPML subscription comes back with the OPML itself. if s.managed || !mine.contains(&s.id) { continue; @@ -1684,7 +1684,7 @@ async fn export_opml( let title = state .ctx .db - .feed_summary(&s.id) + .feed_summary(&s.id).await .ok() .and_then(|sum| sum.title) .unwrap_or_else(|| s.id.clone()); From bd8f6ab85557c36e14b660a15159cb7e589789a5 Mon Sep 17 00:00:00 2001 From: rays <ray.slakinski@gmail.com> Date: Fri, 18 Sep 2026 19:11:24 +0000 Subject: [PATCH 7/9] Docs: the database through SeaORM CLAUDE.md and the architecture notes described the SQL schema and migrate(), both gone: the entities are the schema, create_missing makes what is missing, and hand-written SQL has to run on SQLite and Postgres both. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- CLAUDE.md | 16 +++++++++++----- docs/architecture.md | 17 +++++++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f163bf1..33e6ed8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -125,7 +125,7 @@ Non-trivial logic leaves one runnable check behind. Pure functions (`merge_polic subscriber. Two feeds publishing the same URL means only the first one scanned shows it. * **Read state lives in `entry_state`, per user, and nowhere else.** `entries` had `read`, `flagged` and `position` columns from before accounts; two bugs came from queries still reading them - (retention, and the entry pruner), and `migrate()` now drops them. + (retention, and the entry pruner), and they were dropped in 0.5. * **The catalogue is config.toml; the subscriptions are in the database.** A feed exists once; `subscriptions(user_id, feed_id)` says who wants it and with what settings. OPML children are derived and never written to config. @@ -140,10 +140,16 @@ Non-trivial logic leaves one runnable check behind. Pure functions (`merge_polic * `/api/settings` answering `200` does **not** mean the daemon is well — the web server is a different task. `ipx status` checks the control socket and the database; to see the worker getting through its jobs, watch for `scan complete` in the log. -* **Every `ipx` command runs `migrate()` when it opens the database**, the healthcheck's - `ipx status` included. A migration that rewrites a big table (`DROP COLUMN`) takes seconds on - production, and a command run meanwhile fails with `migrating schema`. It changes nothing; wait - for `daemon started` in the log. Copy `state.db` aside before deploying one. +* **The database goes through SeaORM, and the entities in `src/entity.rs` are the schema.** + `Db::open` creates any missing table or index from them (`create_missing`), on every `ipx` + command, the healthcheck's `ipx status` included, so it must never write when nothing is + missing: SeaORM's experimental schema sync dropped and remade an index on every open, the + write lock that took made `ipx status` time out behind a busy daemon, and it was removed for + it. A new column on an existing table needs its own `ALTER`; nothing adds one for you. +* **SQL written by hand in `db.rs` has to run on SQLite and Postgres both** (issue #18): `$1` + parameters, bound only if used; `ON CONFLICT`, not `INSERT OR IGNORE`; yes/no columns tested + as themselves (`NOT coalesce(s.read, false)`) and written as `true`/`false`, never compared to + 1; no `rowid`, `GLOB` or `UPDATE OR IGNORE`. `Args` in `db.rs` builds the parameters. ## House style diff --git a/docs/architecture.md b/docs/architecture.md index a5357dd..c675006 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,8 @@ it to a running daemon. |---|---|---| | `src/main.rs` | CLI, dispatch, scan loop, download policy | `iPXAgent.py` | | `src/config.rs` | TOML load/save, `General`/`Feed`/`Web`, intervals, slugs | `iPXSettings.py`, `feeds.plist` | -| `src/db.rs` | SQLite schema, migrations, every query | `.ipxd` plists, `history.dat`, `qmcache.dat` | +| `src/db.rs` | Every query, through SeaORM; creates missing tables | `.ipxd` plists, `history.dat`, `qmcache.dat` | +| `src/entity.rs` | The tables, as SeaORM entities: the schema | — | | `src/feed.rs` | Conditional GET, RSS/Atom/OPML parsing | `FeedData.__getFeed/__getEntries` | | `src/download.rs` | Streaming download, naming, type sniffing, placement | `iPXDownloader.getFile` | | `src/torrent.rs` | librqbit session, seeding limits, stall abort | vendored BitTorrent 4.2.1 | @@ -65,14 +66,14 @@ entry_state user_id, feed_id, guid, read, flagged, position ``` Read state is `entry_state` alone. `entries` had `read`, `flagged` and `position` columns from -before accounts; two bugs came from queries still reading them, and `migrate()` drops them from an -older database. +before accounts; two bugs came from queries still reading them, and they were dropped in 0.5. -Schema changes: add the table or column to `SCHEMA`. `CREATE TABLE IF NOT EXISTS` leaves a table -that already exists alone, so a new column on one also goes in `migrate()`'s `wanted` list, and a -retired one in its `retired` list; both are checked with `PRAGMA table_info`. Columns from before -0.3.0, the oldest version an upgrade may start from, need no entry. `Db::memory()` runs the same -path as `Db::open`, so a migration cannot pass the tests while missing in production. +Schema changes: the tables are the entities in `src/entity.rs`, and `Db::open` creates whatever +table or index a database is missing from them (`db::create_missing`), with `IF NOT EXISTS`. It +never alters a table that exists, so a new column on one needs its own `ALTER` in +`create_missing`, or `sea-orm-migration` once there are several. `Db::memory()` builds its +database the same way, so the tests run on the schema production gets. A database from before +0.7 takes its last columns from the old `migrate()`, so it upgrades through a 0.7 release first. ## Control socket From bc53e0f730611da119df94799fe8a46af93048d1 Mon Sep 17 00:00:00 2001 From: rays <ray.slakinski@gmail.com> Date: Fri, 18 Sep 2026 20:28:50 +0000 Subject: [PATCH 8/9] Postgres: pick the database by URL, copy-db, tests on both - IPX_DATABASE_URL (postgres://...) picks the database; unset, it is the SQLite file as before. Passwords are taken out of anything logged. - `ipx copy-db <state.db>` copies every table into the empty database the URL names, in one transaction, and moves the id counters past the copied ids. A copy of production went across in 14s with every count and column fingerprint identical. - With IPX_TEST_DATABASE_URL set, each test gets a Postgres schema of its own; all 79 pass on both databases. Fixtures write booleans as true/false. - Sorts say where an item with no value goes (NULLS FIRST going up, LAST going down): SQLite counts NULL as smallest, Postgres as largest, so "largest first" on Postgres led with every item that has no file. Tested on both. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- src/db.rs | 204 ++++++++++++++++++++++++++++++++++++++--------- src/main.rs | 21 ++++- src/retention.rs | 16 ++-- 3 files changed, 193 insertions(+), 48 deletions(-) diff --git a/src/db.rs b/src/db.rs index aa9f478..91fa768 100644 --- a/src/db.rs +++ b/src/db.rs @@ -67,14 +67,64 @@ async fn create_missing(orm: &sea_orm::DatabaseConnection) -> Result<()> { Ok(()) } +/// One table's rows, a page at a time in primary-key order, from one database to another. +async fn copy_table<E>(from: &sea_orm::DatabaseConnection, to: &impl ConnectionTrait) -> Result<u64> +where + E: EntityTrait, + E::Model: sea_orm::IntoActiveModel<E::ActiveModel> + Send + Sync, + E::ActiveModel: ActiveModelTrait<Entity = E> + Send, +{ + use sea_orm::{IntoActiveModel, Iterable, PrimaryKeyToColumn}; + let mut query = E::find(); + for key in E::PrimaryKey::iter() { + query = query.order_by_asc(key.into_column()); + } + let mut pages = query.paginate(from, 1000); + let mut n = 0; + while let Some(rows) = pages.fetch_and_next().await? { + n += rows.len() as u64; + // reset_all: every column written, the primary key included, not just the changed ones. + E::insert_many(rows.into_iter().map(|m| m.into_active_model().reset_all())) + .exec_without_returning(to) + .await?; + } + Ok(n) +} + +/// Where the database is: IPX_DATABASE_URL, a postgres:// URL, when it is set; otherwise the +/// SQLite file in the data directory, as it has always been. +pub fn location() -> String { + std::env::var("IPX_DATABASE_URL") + .ok() + .filter(|u| !u.trim().is_empty()) + .unwrap_or_else(|| crate::config::data_dir().join("state.db").display().to_string()) +} + +/// A URL fit for a log or an error: the password taken out. +fn redact(url: &str) -> String { + match (url.find("://"), url.rfind('@')) { + (Some(s), Some(at)) if at > s => match url[s + 3..at].find(':') { + Some(c) => format!("{}:***{}", &url[..s + 3 + c], &url[at..]), + None => url.to_owned(), + }, + _ => url.to_owned(), + } +} + +fn is_postgres(location: &str) -> bool { + location.starts_with("postgres://") || location.starts_with("postgresql://") +} + /// sqlx's defaults for SQLite are what ipx wants: foreign keys on, and a five-second wait for a /// lock, which is what rusqlite was set to. -async fn connect(path: &Path) -> Result<sea_orm::DatabaseConnection> { - let mut opts = sea_orm::ConnectOptions::new(format!("sqlite://{}?mode=rwc", path.display())); +async fn connect(location: &str) -> Result<sea_orm::DatabaseConnection> { + let url = + if is_postgres(location) { location.to_owned() } else { format!("sqlite://{location}?mode=rwc") }; + let mut opts = sea_orm::ConnectOptions::new(url); opts.sqlx_logging(false); sea_orm::Database::connect(opts) .await - .with_context(|| format!("opening {}", path.display())) + .with_context(|| format!("opening {}", redact(location))) } @@ -175,18 +225,23 @@ pub struct FeedSummary { } impl Db { - pub async fn open(path: &Path) -> Result<Self> { - if let Some(dir) = path.parent() { - std::fs::create_dir_all(dir) - .with_context(|| format!("creating {}", dir.display()))?; + /// Opens the database at `location` (see `location()`): a postgres:// URL, or a SQLite file. + pub async fn open(location: &str) -> Result<Self> { + if !is_postgres(location) + && let Some(dir) = Path::new(location).parent() + { + std::fs::create_dir_all(dir).with_context(|| format!("creating {}", dir.display()))?; } - let orm = connect(path).await?; - // WAL, so the healthcheck's `ipx status` reads while the daemon writes. It is a setting of - // the file, kept once made, and making it takes a lock that cannot wait out a busy - // daemon, so it is made only when the file is not already in WAL. - let mode = orm.query_one_raw(Statement::from_string(orm.get_database_backend(), "PRAGMA journal_mode")).await?; - if mode.and_then(|r| r.try_get_by_index::<String>(0).ok()).as_deref() != Some("wal") { - orm.execute_unprepared("PRAGMA journal_mode = WAL").await.context("switching to WAL")?; + let orm = connect(location).await?; + if !is_postgres(location) { + // WAL, so the healthcheck's `ipx status` reads while the daemon writes. It is a + // setting of the file, kept once made, and making it takes a lock that cannot wait out + // a busy daemon, so it is made only when the file is not already in WAL. + let mode = + orm.query_one_raw(Statement::from_string(orm.get_database_backend(), "PRAGMA journal_mode")).await?; + if mode.and_then(|r| r.try_get_by_index::<String>(0).ok()).as_deref() != Some("wal") { + orm.execute_unprepared("PRAGMA journal_mode = WAL").await.context("switching to WAL")?; + } } create_missing(&orm).await?; Ok(Self { @@ -196,17 +251,41 @@ impl Db { }) } - /// A fresh database for a test, in a file of its own: two connections to one ":memory:" - /// would be two databases. Its schema comes from the entities, as a real one's does. + /// A fresh, empty database for a test, its schema made from the entities as a real one's is. + /// A SQLite file of its own (two connections to one ":memory:" would be two databases); or, + /// with IPX_TEST_DATABASE_URL set, a Postgres schema of its own on that database, so the + /// same tests prove the SQL on both. #[cfg(test)] pub async fn memory() -> Result<Self> { static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); - let path = std::env::temp_dir().join(format!( - "ipx-test-{}-{}.db", - std::process::id(), - N.fetch_add(1, std::sync::atomic::Ordering::Relaxed) - )); - let orm = connect(&path).await?; + let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let pid = std::process::id(); + if let Some(url) = std::env::var("IPX_TEST_DATABASE_URL").ok().filter(|u| !u.is_empty()) { + let admin = connect(&url).await?; + // Schemas an earlier run left behind: anything of ours not made by this process. + let old = admin + .query_all_raw(Statement::from_string( + admin.get_database_backend(), + format!( + r"SELECT nspname FROM pg_namespace + WHERE nspname LIKE 'ipxt\_%' AND nspname NOT LIKE 'ipxt\_{pid}\_%'" + ), + )) + .await?; + for r in old { + let name: String = r.try_get_by_index(0)?; + admin.execute_unprepared(&format!("DROP SCHEMA IF EXISTS {name} CASCADE")).await?; + } + let schema = format!("ipxt_{pid}_{n}"); + admin.execute_unprepared(&format!("CREATE SCHEMA {schema}")).await?; + // Every connection in the pool starts in it, so each test sees only its own tables. + let sep = if url.contains('?') { '&' } else { '?' }; + let orm = connect(&format!("{url}{sep}options=-c%20search_path%3D{schema}")).await?; + create_missing(&orm).await?; + return Ok(Self { orm, tmp: None }); + } + let path = std::env::temp_dir().join(format!("ipx-test-{pid}-{n}.db")); + let orm = connect(&path.display().to_string()).await?; create_missing(&orm).await?; Ok(Self { orm, tmp: Some(path) }) } @@ -698,7 +777,10 @@ pub fn order_sql(col: &str, dir: &str, pinned_first: bool) -> String { "size" => "(SELECT max(x.length) FROM enclosures x WHERE x.feed_id = e.feed_id AND x.guid = e.guid)", _ => "coalesce(e.published, e.first_seen)", }; - let dir = if dir == "asc" { "ASC" } else { "DESC" }; + // Where an item with no value goes, said outright: SQLite counts a NULL as the smallest value + // and Postgres as the largest, so "largest first" on Postgres opened with every item that has + // no file. NULLS FIRST going up and LAST going down keeps what SQLite did. + let dir = if dir == "asc" { "ASC NULLS FIRST" } else { "DESC NULLS LAST" }; let pins = if pinned_first && col != "kept" { "coalesce(s.flagged, false) DESC, " } else { "" }; // The guid breaks what ties remain: SQLite's rowid did, and Postgres has none. format!("{pins}{expr} {dir}, coalesce(e.published, e.first_seen) DESC, e.guid DESC") @@ -882,6 +964,44 @@ impl Db { Ok(()) } + // ---- moving to another database ---- + + /// Copies every row of `from` into this database, which must be empty: the move from the + /// SQLite file to Postgres (issue #18). One transaction, so a copy that fails part-way leaves + /// nothing behind and can simply be run again. Returns each table's row count. + pub async fn copy_from(&self, from: &Db) -> Result<Vec<(&'static str, u64)>> { + use crate::entity::*; + use sea_orm::TransactionTrait; + let held = users::Entity::find().count(&self.orm).await? + + feeds::Entity::find().count(&self.orm).await? + + entries::Entity::find().count(&self.orm).await?; + anyhow::ensure!(held == 0, "the database being copied into already holds rows; it has to be empty"); + let tx = self.orm.begin().await?; + // Users first: the tables that belong to a person refer to them. + let counts = vec![ + ("users", copy_table::<users::Entity>(&from.orm, &tx).await?), + ("feeds", copy_table::<feeds::Entity>(&from.orm, &tx).await?), + ("entries", copy_table::<entries::Entity>(&from.orm, &tx).await?), + ("enclosures", copy_table::<enclosures::Entity>(&from.orm, &tx).await?), + ("subscriptions", copy_table::<subscriptions::Entity>(&from.orm, &tx).await?), + ("entry_state", copy_table::<entry_state::Entity>(&from.orm, &tx).await?), + ("sessions", copy_table::<sessions::Entity>(&from.orm, &tx).await?), + ]; + if self.orm.get_database_backend() == sea_orm::DbBackend::Postgres { + // The copied ids came with the rows; the counters that hand out new ones start past + // them, or the next account or file would collide with one copied. + for table in ["users", "enclosures"] { + tx.execute_unprepared(&format!( + "SELECT setval(pg_get_serial_sequence('{table}', 'id'), \ + coalesce((SELECT max(id) FROM {table}), 0) + 1, false)" + )) + .await?; + } + } + tx.commit().await?; + Ok(counts) + } + // ---- hand-written SQL ---- // // For what reads better as SQL than as a query builder: joins, sums, upserts. Written to @@ -1598,7 +1718,7 @@ mod tests { async fn every_sort_column_runs_and_orders_both_ways() { let db = Db::memory().await.unwrap(); db.exec_for_test( - "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1); + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true); INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(1,'g'); INSERT INTO feeds (id, url, title) VALUES ('f','u','Zebra'),('g','v','Aardvark'); INSERT INTO entries (feed_id, guid, title, first_seen) VALUES @@ -1620,6 +1740,14 @@ mod tests { assert_eq!(order("feed", "asc").await, ["b", "c", "a"], "Aardvark, then Zebra's newest first"); assert_eq!(order("type", "asc").await, ["a", "b", "c"], "audio, image, video"); assert_eq!(order("size", "desc").await, ["c", "a", "b"]); + // An item with no file has no size or type: last going down, first going up, on both + // databases (they disagree about NULL unless told). + db.exec_for_test("INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('f','n','no file',50);") + .await + .unwrap(); + assert_eq!(order("size", "desc").await.last().map(String::as_str), Some("n")); + assert_eq!(order("type", "asc").await.first().map(String::as_str), Some("n")); + db.exec_for_test("DELETE FROM entries WHERE guid = 'n';").await.unwrap(); assert_eq!(order("published", "desc").await, ["c", "b", "a"]); db.set_entry_flag(1, "f", "a", EntryFlag::Flagged, true).await.unwrap(); assert_eq!(order("kept", "desc").await[0], "a"); @@ -1643,7 +1771,7 @@ mod tests { async fn deleting_a_shared_file_asks_about_everyone_else() { let db = Db::memory().await.unwrap(); db.exec_for_test( - "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0),(3,'kit',0); + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false),(3,'kit',false); INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f'),(3,'f'); INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f','a',0); INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES @@ -1669,7 +1797,7 @@ mod tests { async fn read_state_belongs_to_one_person() { let db = Db::memory().await.unwrap(); db.exec_for_test( - "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false); INSERT INTO entries (feed_id, guid, title, first_seen) VALUES ('f','a','One',100),('f','b','Two',200);", ).await @@ -1751,7 +1879,7 @@ mod tests { // Cutting this along with the dead read columns left the browser suite's admin with an // empty sidebar: it is how a fresh install's first account gets config.toml's feeds. let db = Db::memory().await.unwrap(); - db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',1);").await + db.exec_for_test("INSERT INTO users (id, name, is_admin) VALUES (1,'admin',true);").await .unwrap(); let subs = async || db.i64s_for_test("SELECT count(*) FROM subscriptions").await[0]; let catalogue = ["a".to_string(), "b".to_string()]; @@ -1781,19 +1909,19 @@ mod tests { INSERT INTO enclosures (id, feed_id, guid, url, path, state) VALUES (1,'f','b','u1','/tmp/b','done'); -- Read and starred belong to a person now, so say which one. - INSERT INTO users (id, name, is_admin) VALUES (7,'reader',1); + INSERT INTO users (id, name, is_admin) VALUES (7,'reader',true); INSERT INTO entry_state (user_id, feed_id, guid, read, flagged, position) VALUES - (7,'f','b',1,0,0), - (7,'f','c',1,1,0), + (7,'f','b',true,false,0), + (7,'f','c',true,true,0), -- Started, length unknown: this is Currently Listening. - (7,'f','d',0,0,42), + (7,'f','d',false,false,42), -- 42 of 45 seconds is past the 90% the player calls finished. - (7,'f','e',1,0,42), + (7,'f','e',true,false,42), -- Barely touched (opened, closed within seconds): not Currently Listening. - (7,'f','g',0,0,3), + (7,'f','g',false,false,3), -- Opened, and so read, but 42 of 900 seconds in: still Currently Listening. -- Filtering on read hid exactly these (issue #14). - (7,'f','h',1,0,42);", + (7,'f','h',true,false,42);", ).await .unwrap(); @@ -1877,13 +2005,13 @@ mod tests { async fn a_show_takes_over_what_its_creator_held() { let db = Db::memory().await.unwrap(); db.exec_for_test( - "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1); + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true); INSERT INTO enclosures (id, feed_id, guid, url, state, path, last_error) VALUES (1,'creator','a','u1','done','/x/a.mp3',NULL), (2,'creator','b','u2','skipped',NULL,'explicit'), (3,'creator','c','u3','skipped',NULL,'explicit'), (4,'other','d','u4','skipped',NULL,'torrents disabled'); - INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',1);", + INSERT INTO entry_state (user_id, feed_id, guid, read) VALUES (1,'creator','a',true);", ).await .unwrap(); db.adopt("creator", "show", &[("a", "u1"), ("b", "u2"), ("d", "u4")]).await.unwrap(); @@ -1909,9 +2037,9 @@ mod tests { async fn a_feed_in_a_group_follows_your_settings_on_the_group() { let db = Db::memory().await.unwrap(); db.exec_for_test( - "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false); INSERT INTO subscriptions (user_id, feed_id, allow_explicit) VALUES - (1,'group',1),(1,'show',NULL),(2,'group',1),(2,'show',0);", + (1,'group',true),(1,'show',NULL),(2,'group',true),(2,'show',false);", ).await .unwrap(); let explicit = async |group| -> Vec<Option<bool>> { diff --git a/src/main.rs b/src/main.rs index 595c3a4..9bcbd12 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,6 +37,12 @@ struct Cli { enum Command { /// Show configured feeds and their state List, + /// Copy everything from a SQLite state.db into the database IPX_DATABASE_URL names, which + /// must be empty: the one-off move to Postgres + CopyDb { + /// The SQLite file to copy from + from: PathBuf, + }, /// Scan feeds for new entries Fetch { /// Only this feed id @@ -180,7 +186,7 @@ async fn main() -> Result<()> { let config_path = cli.config.clone().unwrap_or_else(config::config_path); let cfg = config::Config::load(&config_path)?; - let db = db::Db::open(&config::data_dir().join("state.db")).await?; + let db = db::Db::open(&db::location()).await?; // A daemon owns the state; don't have two processes downloading the same thing. let wire_cmd = match &cli.command { @@ -195,7 +201,8 @@ async fn main() -> Result<()> { | Command::Add { .. } | Command::Rm { .. } | Command::Import { .. } - | Command::Export { .. } => None, + | Command::Export { .. } + | Command::CopyDb { .. } => None, }; if let Some(cmd) = &wire_cmd && !cli.local @@ -229,6 +236,7 @@ async fn main() -> Result<()> { Command::User { cmd } => user_cmd(&ctx, cmd).await, Command::Import { file } => import(&ctx, &config_path, &file).await, Command::Export { file } => export(&ctx, &file).await, + Command::CopyDb { from } => copy_db(&ctx, &from).await, _ => run(&ctx, wire_cmd.expect("only List and Daemon have no wire form")).await, } } @@ -770,6 +778,15 @@ pub fn collect_outlines(outlines: &[opml::Outline], out: &mut Vec<(String, Strin } } +async fn copy_db(ctx: &Ctx, from: &std::path::Path) -> Result<()> { + anyhow::ensure!(from.exists(), "{} does not exist", from.display()); + let source = db::Db::open(&from.display().to_string()).await?; + for (table, n) in ctx.db.copy_from(&source).await? { + println!("{table:14} {n}"); + } + Ok(()) +} + async fn export(ctx: &Ctx, file: &std::path::Path) -> Result<()> { let mut doc = opml::OPML::default(); doc.head = Some(opml::Head { diff --git a/src/retention.rs b/src/retention.rs index e58f2eb..009f7cd 100644 --- a/src/retention.rs +++ b/src/retention.rs @@ -154,7 +154,7 @@ mod tests { // One file serves both subscribers, so it takes both of them to release it. let db = Db::memory().await.unwrap(); db.exec_for_test( - "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1),(2,'sam',0); + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true),(2,'sam',false); INSERT INTO subscriptions (user_id, feed_id) VALUES (1,'f'),(2,'f'); INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f', 'keep', 0), @@ -163,11 +163,11 @@ mod tests { ('f', 'read', 0); -- Starred by one of the two, so it stays whatever the other thinks. INSERT INTO entry_state (user_id, feed_id, guid, read, flagged) VALUES - (1, 'f', 'keep', 1, 1), - (2, 'f', 'keep', 1, 0), - (1, 'f', 'half', 1, 0), - (1, 'f', 'read', 1, 0), - (2, 'f', 'read', 1, 0); + (1, 'f', 'keep', true, true), + (2, 'f', 'keep', true, false), + (1, 'f', 'half', true, false), + (1, 'f', 'read', true, false), + (2, 'f', 'read', true, false); INSERT INTO enclosures (id, feed_id, guid, url, path, bytes_done, state, downloaded_at) VALUES (1, 'f', 'keep', 'u1', '/tmp/keep', 10, 'done', 10), (2, 'f', 'half', 'u2', '/tmp/half', 10, 'done', 20), @@ -189,8 +189,8 @@ mod tests { async fn prune_keeps_entries_that_still_have_a_file() { let db = Db::memory().await.unwrap(); db.exec_for_test( - "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',1); - INSERT INTO entry_state (user_id, feed_id, guid, flagged) VALUES (1,'f','flagged',1); + "INSERT INTO users (id, name, is_admin) VALUES (1,'ray',true); + INSERT INTO entry_state (user_id, feed_id, guid, flagged) VALUES (1,'f','flagged',true); INSERT INTO entries (feed_id, guid, first_seen) VALUES ('f', 'has-file', 100), ('f', 'no-file', 100), From c1c06229c8d1b210b0e40c42942dfb35fbf2feea Mon Sep 17 00:00:00 2001 From: rays <ray.slakinski@gmail.com> Date: Fri, 18 Sep 2026 21:42:27 +0000 Subject: [PATCH 9/9] Docs: Postgres in production Where the database now is and how to reach it, IPX_DATABASE_URL and IPX_TEST_DATABASE_URL, copy-db, backups, and the title sort on Postgres. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- CHANGELOG.md | 9 ++++++++- CLAUDE.md | 3 ++- docs/configuration.md | 8 ++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb1b8da..0adf11c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- ipx can keep its data in Postgres: set `IPX_DATABASE_URL` to a `postgres://` URL. Without it, + it is the SQLite `state.db` as before. `ipx copy-db <state.db>` moves an existing database + across, everything in one go. + ### Changed - The database is reached through SeaORM, on the way to Postgres (issue #18); it is still the - same SQLite file, and nothing you see changes. A database from before 0.7 has to be opened by a + same SQLite file, and nothing you see changes. On Postgres, sorting by title or feed follows + the language's order (an accented letter beside the plain one) rather than raw bytes. A database from before 0.7 has to be opened by a 0.7 release first, which brings its tables up to date. - A pinned item sits at the top of its list, above everything else in whatever order you sort by, and moves there the moment you pin it. Sorting by the pin column itself still goes both diff --git a/CLAUDE.md b/CLAUDE.md index 33e6ed8..026d229 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,8 @@ Arcane project `content`: `/mnt/fast/arcane/projects/content/compose.yaml`. That |---|---|---| | Image | `192.168.1.130:5000/ipodderx:latest` | | | Config | `/mnt/fast/appdata/ipodderx/config.toml` | `/config/config.toml` | -| Database | `/mnt/user/ipodderx/state.db` | `/data/state.db` | +| Database | Postgres 18, database `ipodderx`, login `ipodderx`, on the `postgres` container of the Arcane project `databases` (`192.168.1.130:5433`). The URL is in `/mnt/fast/appdata/ipodderx/database.env` (root-only), passed to the container as `IPX_DATABASE_URL` | | +| Old database | `/mnt/user/ipodderx/state.db`, SQLite, used until the move to Postgres on 2026-09-18 and kept for rollback | `/data/state.db` | | Downloads | `/mnt/user/ipodderx/downloads` | `/downloads` | | Web UI | `192.168.1.130:8099`, also `ipodderx.sdf1.net` via a Cloudflare tunnel | `0.0.0.0:8099` | | Sign-in via the tunnel | Cloudflare Access app `ipodderx`, with Authentik as its identity provider; see [docs/sso.md](docs/sso.md) | trusts `Cf-Access-Authenticated-User-Email` from `192.168.16.1`, the `content_default` gateway | diff --git a/docs/configuration.md b/docs/configuration.md index ced9093..7deb090 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,8 +11,10 @@ effect without a restart. Default location `$XDG_CONFIG_HOME/ipx/config.toml` | Control socket | `$XDG_RUNTIME_DIR/ipx.sock` | `[general] socket` | | Downloads | `[general] download_dir` | — | -`~` is expanded in paths. The database is SQLite in WAL mode; back it up by copying `state.db` -while the daemon is stopped, or with `sqlite3 state.db .backup`. +`~` is expanded in paths. The database is SQLite in WAL mode unless `IPX_DATABASE_URL` names a +Postgres database instead. Back SQLite up by copying `state.db` while the daemon is stopped, or +with `sqlite3 state.db .backup`; back Postgres up with `pg_dump`. `ipx copy-db <state.db>` copies a +SQLite database into the empty Postgres one `IPX_DATABASE_URL` names. ## `[general]` @@ -116,6 +118,8 @@ they are re-derived on every scan. Editing one in the UI promotes it to a real c |---|---| | `IPX_CONFIG` | Config file path | | `IPX_DATA_DIR` | Directory holding `state.db` | +| `IPX_DATABASE_URL` | A `postgres://user:password@host:port/database` URL: use that database instead of `state.db` | +| `IPX_TEST_DATABASE_URL` | For `cargo test`: run the database tests on this Postgres database too, each in a schema of its own | | `IPX_LOG` | What reaches stderr (`ipx=debug`, `ipx::scan=debug`, …) | | `IPX_UI_LOG` | What the in-process log buffer captures for the UI's Log view | | `http_proxy` / `https_proxy` | Honoured for feed and enclosure fetches |