diff options
25 files changed, 907 insertions, 964 deletions
diff --git a/Cargo.lock b/Cargo.lock index aa9de48..765d4fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,12 +159,6 @@ dependencies = [ ] [[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -294,7 +288,7 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.1" dependencies = [ "serde", ] @@ -646,6 +640,36 @@ dependencies = [ ] [[package]] +name = "curl" +version = "0.4.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2d5c8f48d9c0c23250e52b55e82a6ab4fdba6650c931f5a0a57a43abda812b" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "windows-sys 0.59.0", +] + +[[package]] +name = "curl-sys" +version = "0.4.82+curl-8.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4d63638b5ec65f1a4ae945287b3fd035be4554bbaf211901159c9a2a74fb5be" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "windows-sys 0.59.0", +] + +[[package]] name = "der" version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -734,15 +758,6 @@ dependencies = [ ] [[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] name = "endian-type" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -879,12 +894,6 @@ dependencies = [ ] [[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1079,25 +1088,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] -name = "h2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" -dependencies = [ - "atomic-waker", - "bytes 1.10.1", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] name = "half" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1179,124 +1169,6 @@ dependencies = [ ] [[package]] -name = "http" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" -dependencies = [ - "bytes 1.10.1", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes 1.10.1", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes 1.10.1", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "hyper" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" -dependencies = [ - "bytes 1.10.1", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", -] - -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes 1.10.1", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" -dependencies = [ - "base64", - "bytes 1.10.1", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "system-configuration", - "tokio", - "tower-service", - "tracing", - "windows-registry", -] - -[[package]] name = "iana-time-zone" version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1458,22 +1330,6 @@ dependencies = [ ] [[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" -dependencies = [ - "memchr", - "serde", -] - -[[package]] name = "is-macro" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1702,7 +1558,7 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libmpv2" -version = "1.5.0" +version = "1.6.1" dependencies = [ "crossbeam", "libmpv2-sys", @@ -1712,7 +1568,7 @@ dependencies = [ [[package]] name = "libmpv2-sys" -version = "1.5.0" +version = "1.6.1" dependencies = [ "bindgen", ] @@ -1749,6 +1605,18 @@ dependencies = [ ] [[package]] +name = "libz-sys" +version = "1.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] name = "linux-raw-sys" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1901,12 +1769,6 @@ dependencies = [ ] [[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1943,23 +1805,6 @@ dependencies = [ ] [[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] name = "nibble_vec" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2510,48 +2355,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] -name = "reqwest" -version = "0.12.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" -dependencies = [ - "base64", - "bytes 1.10.1", - "encoding_rs", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-tls", - "hyper-util", - "js-sys", - "log", - "mime", - "native-tls", - "percent-encoding", - "pin-project-lite", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-native-tls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] name = "result-like" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2573,20 +2376,6 @@ dependencies = [ ] [[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.16", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] name = "rsa" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2693,39 +2482,6 @@ dependencies = [ ] [[package]] -name = "rustls" -version = "0.23.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" -dependencies = [ - "once_cell", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" -dependencies = [ - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] name = "rustpython" version = "0.4.0" source = "git+https://github.com/RustPython/RustPython.git#c968fe0fd9d7466dc9d5d97e973b82ba35e516d8" @@ -2890,6 +2646,8 @@ version = "0.4.0" source = "git+https://github.com/RustPython/RustPython.git#c968fe0fd9d7466dc9d5d97e973b82ba35e516d8" dependencies = [ "glob", + "rustpython-compiler-core", + "rustpython-derive", ] [[package]] @@ -3155,29 +2913,6 @@ dependencies = [ ] [[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.9.1", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] name = "serde" version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3626,15 +3361,6 @@ dependencies = [ ] [[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] name = "synstructure" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3699,7 +3425,7 @@ dependencies = [ [[package]] name = "termsize" -version = "1.5.0" +version = "1.6.1" dependencies = [ "libc", "winapi", @@ -3820,26 +3546,6 @@ dependencies = [ ] [[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] name = "tokio-stream" version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3907,51 +3613,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "bitflags 2.9.1", - "bytes 1.10.1", - "futures-util", - "http", - "http-body", - "iri-string", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] name = "tracing" version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3984,12 +3645,6 @@ dependencies = [ ] [[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] name = "twox-hash" version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4189,12 +3844,6 @@ dependencies = [ ] [[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] name = "url" version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4220,7 +3869,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uu_fmt" -version = "1.5.0" +version = "1.6.1" dependencies = [ "unicode-width", ] @@ -4265,15 +3914,6 @@ dependencies = [ ] [[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4321,19 +3961,6 @@ dependencies = [ ] [[package]] -name = "wasm-bindgen-futures" -version = "0.4.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] name = "wasm-bindgen-macro" version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4366,16 +3993,6 @@ dependencies = [ ] [[package]] -name = "web-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] name = "which" version = "7.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4505,17 +4122,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] -name = "windows-registry" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] name = "windows-result" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4832,11 +4438,11 @@ dependencies = [ [[package]] name = "yt" -version = "1.5.0" +version = "1.6.1" dependencies = [ "anyhow", "blake3", - "bytes 1.5.0", + "bytes 1.6.1", "chrono", "chrono-humanize", "clap", @@ -4865,11 +4471,11 @@ dependencies = [ [[package]] name = "yt_dlp" -version = "1.5.0" +version = "1.6.1" dependencies = [ + "curl", "indexmap", "log", - "reqwest", "rustpython", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 470eb58..56a90fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ members = [ [workspace.package] edition = "2024" -version = "1.5.0" +version = "1.6.1" rust-version = "1.85.0" authors = ["Benedikt Peetz <benedikt.peetz@b-peetz.de>"] repository = "https://git.vhack.eu/soispha/clients/yt" diff --git a/NEWS.md b/NEWS.md index 4e57d7b..44142e4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,109 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. All notable changes to this project will be documented in this file. See [conventional commits](https://www.conventionalcommits.org/) for commit guidelines. - - - +## [v1.6.1](https://git.foss-syndicate.org/soispha/clients/yt/compare/3f6ef87fc31581215cb00d56462b35e07b7a1f28..v1.6.1) - 2025-06-17 +#### Bug Fixes +- **(package)** Set the PYTHONPATH ourselves - ([ea77b89](https://git.foss-syndicate.org/soispha/clients/yt/commit/ea77b898e5dfb2a7900a87a1bb73167a6e1a140c)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp)** Typos in strings - ([987cff2](https://git.foss-syndicate.org/soispha/clients/yt/commit/987cff2b5996cc86069dc1d9cbb0f465c32d391c)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/)** Include the frozen python stdlib - ([3f6ef87](https://git.foss-syndicate.org/soispha/clients/yt/commit/3f6ef87fc31581215cb00d56462b35e07b7a1f28)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/post_processing/dearrow)** Correctly type the `CasualVote` field - ([528c2d4](https://git.foss-syndicate.org/soispha/clients/yt/commit/528c2d4a4842647da3a91a034c810c44ebf9b949)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/post_processors)** Register in python - ([1a6d363](https://git.foss-syndicate.org/soispha/clients/yt/commit/1a6d3639e6fddb731735554d407d1eea77f053c6)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/post_processors/dearrow)** Migrate to curl for api requests - ([0a17001](https://git.foss-syndicate.org/soispha/clients/yt/commit/0a1700131341c5dac55a395ce5ccdac4f8ec0c9e)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Build system +- **(flake)** Teach the flake about the new package.nix location - ([c4bc9fd](https://git.foss-syndicate.org/soispha/clients/yt/commit/c4bc9fdfde2852cc0f5efbb9bed327f16a6fe275)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **({Cargo,flake}.lock)** Update - ([2aaa919](https://git.foss-syndicate.org/soispha/clients/yt/commit/2aaa919101be7a4fa42ac76a5f2f491689319e39)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Miscellaneous Chores +- **(treewide)** Assure that `nix fmt` and `reuse lint` are happy - ([d847968](https://git.foss-syndicate.org/soispha/clients/yt/commit/d847968fab7dc55b30f8a137dbce2bae07112c82)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Refactoring +- **(nix/package)** Avoid the duplicated `package` name - ([9fbbd3e](https://git.foss-syndicate.org/soispha/clients/yt/commit/9fbbd3e71f2d7286e9ef1cbdbdea4020bd511308)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp)** Split the big `lib.rs` file up - ([8d6eb78](https://git.foss-syndicate.org/soispha/clients/yt/commit/8d6eb786ee99e7b0c36736152e30a5f61cd34167)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/logging)** Avoid adding to the `__all__` list - ([e0120c0](https://git.foss-syndicate.org/soispha/clients/yt/commit/e0120c08672009f8d4445eebef8efb22ddae5fb3)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/progress_hook)** Use public api via `__priv` module - ([74ecf0e](https://git.foss-syndicate.org/soispha/clients/yt/commit/74ecf0ea1564343905a96dbd14826700762ec825)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) + +- - - + +## [v1.6.0](https://git.foss-syndicate.org/soispha/clients/yt/compare/07db485f9c5206fbcfe2a5f9db28a9587edc6d2b..v1.6.0) - 2025-06-16 +#### Bug Fixes +- **(libmpv2-sys)** Avoid generating comments, that confuse rustdoc - ([0c0e00d](https://git.foss-syndicate.org/soispha/clients/yt/commit/0c0e00da2c21c4b8325fa6145c808e9df0df0834)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(scripts/mkdb.sh)** Also use the `$DATABASE_URL` variable as source source - ([45e5500](https://git.foss-syndicate.org/soispha/clients/yt/commit/45e55007aa13b1ec24af4c543bc3b8699710301c)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/cli)** Remove duplicated short flag key (help also uses 'h') - ([c1122d6](https://git.foss-syndicate.org/soispha/clients/yt/commit/c1122d6ab31548aff9bf8aaa4a855a771355c8e9)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/download/get_file_size)** Correct deal with `filesize_approx` = Null - ([680f811](https://git.foss-syndicate.org/soispha/clients/yt/commit/680f811adc83554cfbaff56d8b50501786a949e2)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/downloader/progress_hook)** Silence clippy warnings - ([65ba5d7](https://git.foss-syndicate.org/soispha/clients/yt/commit/65ba5d738dcfeaecb398e246e0db5d7c4bf04b99)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/select/selection_file/duration)** Improve the duration parser - ([9e1c1ae](https://git.foss-syndicate.org/soispha/clients/yt/commit/9e1c1aec0548a6482e23ceac4e1265ef8baf8023)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/migrate)** Correct the two to three migration script - ([7694496](https://git.foss-syndicate.org/soispha/clients/yt/commit/7694496efa621466e327b9c00fe1c5cc092ccc1f)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/migrate)** Correctly state the upgrade to the topmost version - ([449c4c2](https://git.foss-syndicate.org/soispha/clients/yt/commit/449c4c26c91400e56e0e685958b825b3f02f4e40)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/migrate)** Improve error messages - ([3a16edd](https://git.foss-syndicate.org/soispha/clients/yt/commit/3a16edde524f881c8955026350243a1b4d54d89b)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/migrate)** Account for the fact that DbVersions::Empty means no Version - ([d1f004c](https://git.foss-syndicate.org/soispha/clients/yt/commit/d1f004ce48caf90ab4f3ec1d0bbb588c9cbf0fe9)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/video_database/set)** Reset the `is_focused` flag - ([07db485](https://git.foss-syndicate.org/soispha/clients/yt/commit/07db485f9c5206fbcfe2a5f9db28a9587edc6d2b)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/subscribe)** Deal with moved url value - ([fb00ecf](https://git.foss-syndicate.org/soispha/clients/yt/commit/fb00ecf745c1bd12e026faabf235a75c2c775a3a)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/update)** Also handle the newly introduced error conditions - ([a7e1a2d](https://git.foss-syndicate.org/soispha/clients/yt/commit/a7e1a2d7475fc1304ef7b33aa2f170f8232bd1d8)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/update)** Correct the progress display in `--grouped` mode - ([35f400c](https://git.foss-syndicate.org/soispha/clients/yt/commit/35f400cebca70325e7e999f15dcaa562dbc78f25)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/update)** Avoid printing all the subscriptions that are not updated - ([810c0d3](https://git.foss-syndicate.org/soispha/clients/yt/commit/810c0d3e75287c15e8baf210f89c807a21d3acee)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/update/video_entry_to_video)** Cast the json objects - ([b6a57c5](https://git.foss-syndicate.org/soispha/clients/yt/commit/b6a57c5cad1ee7df56dad5ccb2317f936e682bbe)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/version)** Use yt_dlp's native python version imply - ([22f74fc](https://git.foss-syndicate.org/soispha/clients/yt/commit/22f74fc43b004045d13b0184ae075dea0ebc8eda)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/watch/playlist)** Workaround terminals, that treat 0 as 1 - ([b3be18a](https://git.foss-syndicate.org/soispha/clients/yt/commit/b3be18a0bfb55135135c9769ac531c098ca4d26c)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/{se,}dowa)** Don't exit completely, if the downloader fails - ([b70dd45](https://git.foss-syndicate.org/soispha/clients/yt/commit/b70dd458615bbad99cf05dbde3dc831a9922ba21)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp)** Avoid writing the json output to disk - ([c8601d6](https://git.foss-syndicate.org/soispha/clients/yt/commit/c8601d67c2dd67ed3ae4465fbf3906fa2cf15a98)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/json_{cast,get})** Improve error reporting - ([c4f8c14](https://git.foss-syndicate.org/soispha/clients/yt/commit/c4f8c14b5636055a2973afe0d5ef6494d97a1a76)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Build system +- **(.envrc)** Also disable ytdlp plugins by default - ([1b8113a](https://git.foss-syndicate.org/soispha/clients/yt/commit/1b8113a72161e5d5f1f7a8328265f8075fc3491a)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(.envrc)** Remove outdated env variables - ([e51139d](https://git.foss-syndicate.org/soispha/clients/yt/commit/e51139da51bbb8725614356bd173d1d66af7f74f)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(cog.toml)** Use the correct remote url - ([4e2aeec](https://git.foss-syndicate.org/soispha/clients/yt/commit/4e2aeec877ec9083de5116bcca8da039389b9f09)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(cog.toml)** Use correct username - ([b9957a2](https://git.foss-syndicate.org/soispha/clients/yt/commit/b9957a2dc50b02f1df8bcb2dc3ddcc3c081b94d3)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(flake)** Document for what the `CLANG_*` env vars are needed - ([d03e537](https://git.foss-syndicate.org/soispha/clients/yt/commit/d03e5374a238dc2b701c259bbd5ade91c6b4a9ff)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(flake)** Remove `flake-utils` - ([d21e1ac](https://git.foss-syndicate.org/soispha/clients/yt/commit/d21e1ac26c5e57f7e5f9cb2fea937b807118187b)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(flake)** Switch to `nixpkgs-unstable-small` - ([1a807d2](https://git.foss-syndicate.org/soispha/clients/yt/commit/1a807d25bd1a47fb81b538a1638514cedb928148)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(flake)** Adapt the dev env to yt_dlp's new dependencies - ([b8682b4](https://git.foss-syndicate.org/soispha/clients/yt/commit/b8682b478a3a2322a370cc8eabf46d20d00e8c37)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(flake)** Add `git-bug` to the devshell - ([ebcd3e1](https://git.foss-syndicate.org/soispha/clients/yt/commit/ebcd3e153e01bd1432b583b2a09569ba2017b8ed)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(package/package.nix)** Update to the new build requirements - ([f590cef](https://git.foss-syndicate.org/soispha/clients/yt/commit/f590cef92da3931fae1607c6964b8125ab2f6307)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(treewide)** Update - ([2380d7d](https://git.foss-syndicate.org/soispha/clients/yt/commit/2380d7d7fdfdda91c26e8027f41aa6788f3590e0)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(treewide)** Update - ([0791777](https://git.foss-syndicate.org/soispha/clients/yt/commit/0791777665fe99d02b5e4aaaa43ca3483712dac9)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(treewide)** Update - ([fa79bd7](https://git.foss-syndicate.org/soispha/clients/yt/commit/fa79bd7eef3824ad208984df9cc7784bdab5ba2b)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(treewide)** Update - ([47754f5](https://git.foss-syndicate.org/soispha/clients/yt/commit/47754f54b978e7ed66ccd29c866fabe28607997e)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **({flake,Cargo}.lock)** Update - ([9c7dfa7](https://git.foss-syndicate.org/soispha/clients/yt/commit/9c7dfa7a8ca71bd5067741917a6f96061290976b)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Documentation +- **(yt/update)** Add comment about the `unsmuggle_url` invocation - ([6c47d93](https://git.foss-syndicate.org/soispha/clients/yt/commit/6c47d93c983b8807032220e107ac2f686abb14e2)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/watch/playlist_handler/client_messages)** Add TODO about `current_exe` - ([13a0621](https://git.foss-syndicate.org/soispha/clients/yt/commit/13a062150e4efaf4b87d9213cf68b5a4eabb0235)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp)** Fix typo in `Cargo.toml`'s description - ([848270e](https://git.foss-syndicate.org/soispha/clients/yt/commit/848270ed0d9ed0409fe563a130e2913d9dfcc897)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Features +- **(yt/cli)** Add support for command line completions - ([e635ee7](https://git.foss-syndicate.org/soispha/clients/yt/commit/e635ee79a4ec0d30dca271cc269fee40150ea821)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/select)** Support a directory selection process - ([e2d5dc6](https://git.foss-syndicate.org/soispha/clients/yt/commit/e2d5dc6a9f000a875c3f2a100f660adc2a43275a)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/select/split)** Make sorting configurable - ([8b644e4](https://git.foss-syndicate.org/soispha/clients/yt/commit/8b644e4e0e058a003984c02d48e829de437145c6)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/status)** Show the percentage of videos that were actually watched - ([ec4e0c9](https://git.foss-syndicate.org/soispha/clients/yt/commit/ec4e0c91d33b2a8c11b71d4cdb1edeaa44ce8247)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/videos)** Validate in DB, that is_focused is UNIQUE - ([cf16b93](https://git.foss-syndicate.org/soispha/clients/yt/commit/cf16b93b563daee88b3bda4b30666b1b0766a8b0)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/update)** Print a nice progress number - ([c04d530](https://git.foss-syndicate.org/soispha/clients/yt/commit/c04d530a1a9e09dd26adc4116959e5481b970bc6)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/update)** Support grouped updates - ([8a42c83](https://git.foss-syndicate.org/soispha/clients/yt/commit/8a42c835a0dd1fcaa3475938d9442199d57acf75)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/update)** Specify subscriptions to update as positional args - ([51bbd90](https://git.foss-syndicate.org/soispha/clients/yt/commit/51bbd90ab1f08c9056c4e5799e3abba568ae75c9)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/videos/list)** Replace the nucleo matcher with a simple `contains` - ([c0a3b61](https://git.foss-syndicate.org/soispha/clients/yt/commit/c0a3b61fb344a5ca86cae1c31d2e42fbe56b6726)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp)** Support a DeArrow post processor - ([ab61a4e](https://git.foss-syndicate.org/soispha/clients/yt/commit/ab61a4e47a955dd4a5dabeef3ade1b85f6576b84)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **({yt/update,yt_dlp})** Use yt_dlp errors again - ([078dfa0](https://git.foss-syndicate.org/soispha/clients/yt/commit/078dfa09a40a384b5cb8cf8cffd9b68cc9678556)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **({yt_dlp,yt})** Migrate from pyo3 to rustpython - ([69145b4](https://git.foss-syndicate.org/soispha/clients/yt/commit/69145b4deed4fe512239a9f88e6af69d3b8c0309)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Miscellaneous Chores +- **(treewide)** Add missing copyright headers - ([fd029a6](https://git.foss-syndicate.org/soispha/clients/yt/commit/fd029a65d43e1eb935b470b88893c16c30c19746)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(treewide)** Migrate to rust edition 2024 - ([8be7171](https://git.foss-syndicate.org/soispha/clients/yt/commit/8be717167ed77f5a1021fa0825b386674c5c1a39)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/wrappers/info_json)** Add additional missing field - ([8ef4cf9](https://git.foss-syndicate.org/soispha/clients/yt/commit/8ef4cf92635003fb79263d22126289d788e34633)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Performance Improvements +- **(yt/update/updater)** Acknowledge, that `yt_dlp` has a sync API - ([ab56f25](https://git.foss-syndicate.org/soispha/clients/yt/commit/ab56f2550d5086ccd1c6981b62081b70743a1f2c)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Refactoring +- **(yt)** Move to `crates/yt` - ([394d4f7](https://git.foss-syndicate.org/soispha/clients/yt/commit/394d4f7d105dadd7b516f198b0d6a9dda2d3f1a5)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt)** Consolidate the multiple ANSI escape code wrapper functions - ([efc35b5](https://git.foss-syndicate.org/soispha/clients/yt/commit/efc35b5bd76bf4e4aab6750ead45713a79e851f9)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/select)** Split the `select::select` function up - ([fb49841](https://git.foss-syndicate.org/soispha/clients/yt/commit/fb49841e1ec14b3ab2de981e439d4f10f5494cf5)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/select/selection_file)** Migrate from `trinitry` to `shlex` - ([56011be](https://git.foss-syndicate.org/soispha/clients/yt/commit/56011be94c09828b104008cb7bf3a19177bc1631)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/migrate)** Factor out duplicated code into macro - ([137339d](https://git.foss-syndicate.org/soispha/clients/yt/commit/137339d1d2924da764c54517fcc6d5d11d46a69d)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/storage/migrate/sql)** Use predictable SQL paths - ([420f9c8](https://git.foss-syndicate.org/soispha/clients/yt/commit/420f9c87abe3a3480a2345cbad5ec427636b2cb5)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp)** Remove the unneeded `async` from the public functions - ([5b5caee](https://git.foss-syndicate.org/soispha/clients/yt/commit/5b5caee512dd82bc5106e69259ba916cd143deda)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/lib)** De-duplicate the info json sanitize code - ([e46ab9b](https://git.foss-syndicate.org/soispha/clients/yt/commit/e46ab9bc8bd4ecc35363e27aea9b5445bc858b2d)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt_dlp/lib)** Explicitly convert python exceptions into an error - ([ada9550](https://git.foss-syndicate.org/soispha/clients/yt/commit/ada9550b02ee13a8378bd2ee27d536b83eec4820)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Style +- **(treewide)** Reformat - ([10b07fa](https://git.foss-syndicate.org/soispha/clients/yt/commit/10b07fa5a4f4080ef5417720b2d15179b72d2fc2)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(treewide)** Adopt rust edition 2024 rustfmt style - ([a78b66e](https://git.foss-syndicate.org/soispha/clients/yt/commit/a78b66ed784cd6f2f97771d9e170c8f8558140b8)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/cli)** Sort the toplevel flags alphabetically - ([77ea1d8](https://git.foss-syndicate.org/soispha/clients/yt/commit/77ea1d8223b57567b448fb973b7240adaab61778)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +- **(yt/select)** Apply clippy's suggestions - ([b4ee42a](https://git.foss-syndicate.org/soispha/clients/yt/commit/b4ee42a62c683632c589f39e6ceac0b48d222e87)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) +#### Tests +- **(yt/cli)** Test the CLI - ([10f9d8b](https://git.foss-syndicate.org/soispha/clients/yt/commit/10f9d8bfd0c84146638cfdaf6b076493f943e650)) - [@bpeetz](https://git.foss-syndicate.org/bpeetz) + +- - - + ## [v1.5.0](https://git.vhack.eu/soispha/clients/yt/compare/2146109725115a9d01cc08ebbe3ef9c533ef1a89..v1.5.0) - 2025-02-22 #### Bug Fixes - **(crates/libmpv2)** Improve the error message for the `RawError` - ([0bd13d5](https://git.vhack.eu/soispha/clients/yt/commit/0bd13d5c26495649dabc23a4fb6b37fe682e3aec)) - [@soispha](https://git.vhack.eu/soispha) diff --git a/crates/yt/src/comments/description.rs b/crates/yt/src/comments/description.rs index e8cb29d..878b573 100644 --- a/crates/yt/src/comments/description.rs +++ b/crates/yt/src/comments/description.rs @@ -17,7 +17,7 @@ use crate::{ }; use anyhow::{Result, bail}; -use yt_dlp::{InfoJson, json_cast}; +use yt_dlp::json_cast; pub async fn description(app: &App) -> Result<()> { let description = get(app).await?; @@ -34,7 +34,7 @@ pub async fn get(app: &App) -> Result<String> { bail!("Could not find a currently playing video!"); }; - let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( + let info_json = get::video_info_json(¤tly_playing_video)?.unreachable( "A currently *playing* must be cached. And thus the info.json should be available", ); diff --git a/crates/yt/src/comments/mod.rs b/crates/yt/src/comments/mod.rs index 876146d..54031a4 100644 --- a/crates/yt/src/comments/mod.rs +++ b/crates/yt/src/comments/mod.rs @@ -15,7 +15,7 @@ use anyhow::{Result, bail}; use comment::{Comment, CommentExt, Comments, Parent}; use output::display_fmt_and_less; use regex::Regex; -use yt_dlp::{InfoJson, json_cast}; +use yt_dlp::json_cast; use crate::{ app::App, @@ -39,7 +39,7 @@ pub async fn get(app: &App) -> Result<Comments> { bail!("Could not find a currently playing video!"); }; - let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( + let info_json = get::video_info_json(¤tly_playing_video)?.unreachable( "A currently *playing* video must be cached. And thus the info.json should be available", ); diff --git a/crates/yt/src/download/download_options.rs b/crates/yt/src/download/download_options.rs index 03c20ba..558adfd 100644 --- a/crates/yt/src/download/download_options.rs +++ b/crates/yt/src/download/download_options.rs @@ -11,7 +11,7 @@ use anyhow::Context; use serde_json::{Value, json}; -use yt_dlp::{YoutubeDL, YoutubeDLOptions}; +use yt_dlp::{YoutubeDL, options::YoutubeDLOptions}; use crate::{app::App, storage::video_database::YtDlpOptions}; diff --git a/crates/yt/src/select/cmds/add.rs b/crates/yt/src/select/cmds/add.rs index 387b3a1..2fff298 100644 --- a/crates/yt/src/select/cmds/add.rs +++ b/crates/yt/src/select/cmds/add.rs @@ -20,7 +20,7 @@ use crate::{ use anyhow::{Context, Result, bail}; use log::{error, warn}; use url::Url; -use yt_dlp::{InfoJson, YoutubeDL, json_cast, json_get}; +use yt_dlp::{YoutubeDL, info_json::InfoJson, json_cast, json_get}; #[allow(clippy::too_many_lines)] pub(super) async fn add( diff --git a/crates/yt/src/storage/subscriptions.rs b/crates/yt/src/storage/subscriptions.rs index 6c0d08a..1ab0d72 100644 --- a/crates/yt/src/storage/subscriptions.rs +++ b/crates/yt/src/storage/subscriptions.rs @@ -17,7 +17,7 @@ use anyhow::Result; use log::debug; use sqlx::query; use url::Url; -use yt_dlp::YoutubeDLOptions; +use yt_dlp::options::YoutubeDLOptions; use crate::{app::App, unreachable::Unreachable}; diff --git a/crates/yt/src/storage/video_database/get/mod.rs b/crates/yt/src/storage/video_database/get/mod.rs index 0456cd3..e76131e 100644 --- a/crates/yt/src/storage/video_database/get/mod.rs +++ b/crates/yt/src/storage/video_database/get/mod.rs @@ -18,7 +18,7 @@ use anyhow::{Context, Result, bail}; use blake3::Hash; use log::{debug, trace}; use sqlx::query; -use yt_dlp::InfoJson; +use yt_dlp::info_json::InfoJson; use crate::{ app::App, diff --git a/crates/yt/src/subscribe/mod.rs b/crates/yt/src/subscribe/mod.rs index 7ac0be4..a965ac0 100644 --- a/crates/yt/src/subscribe/mod.rs +++ b/crates/yt/src/subscribe/mod.rs @@ -16,7 +16,7 @@ use futures::FutureExt; use log::warn; use tokio::io::{AsyncBufRead, AsyncBufReadExt}; use url::Url; -use yt_dlp::{YoutubeDLOptions, json_get}; +use yt_dlp::{json_get, options::YoutubeDLOptions}; use crate::{ app::App, diff --git a/crates/yt/src/update/mod.rs b/crates/yt/src/update/mod.rs index d866882..7f9bee7 100644 --- a/crates/yt/src/update/mod.rs +++ b/crates/yt/src/update/mod.rs @@ -15,7 +15,7 @@ use anyhow::{Context, Ok, Result}; use chrono::{DateTime, Utc}; use log::warn; use url::Url; -use yt_dlp::{InfoJson, json_cast, json_get}; +use yt_dlp::{info_json::InfoJson, json_cast, json_get}; use crate::{ app::App, diff --git a/crates/yt/src/update/updater.rs b/crates/yt/src/update/updater.rs index 60e9855..75d12dc 100644 --- a/crates/yt/src/update/updater.rs +++ b/crates/yt/src/update/updater.rs @@ -19,7 +19,10 @@ use futures::{StreamExt, future::join_all, stream}; use log::{Level, debug, error, log_enabled}; use serde_json::json; use tokio_util::task::LocalPoolHandle; -use yt_dlp::{InfoJson, PythonError, YoutubeDLOptions, json_cast, json_get, process_ie_result}; +use yt_dlp::{ + info_json::InfoJson, json_cast, json_get, options::YoutubeDLOptions, process_ie_result, + python_error::PythonError, +}; use crate::{ ansi_escape_codes::{clear_whole_line, move_to_col}, diff --git a/crates/yt/src/version/mod.rs b/crates/yt/src/version/mod.rs index 9a91f3b..95660c0 100644 --- a/crates/yt/src/version/mod.rs +++ b/crates/yt/src/version/mod.rs @@ -10,7 +10,7 @@ use anyhow::{Context, Result}; use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; -use yt_dlp::YoutubeDLOptions; +use yt_dlp::options::YoutubeDLOptions; use crate::{config::Config, storage::migrate::get_version_db}; diff --git a/crates/yt_dlp/Cargo.toml b/crates/yt_dlp/Cargo.toml index 81e1412..4f62eec 100644 --- a/crates/yt_dlp/Cargo.toml +++ b/crates/yt_dlp/Cargo.toml @@ -22,13 +22,14 @@ rust-version.workspace = true publish = true [dependencies] +curl = "0.4.48" indexmap = { version = "2.9.0", default-features = false } log.workspace = true -reqwest = { version = "0.12.20", features = ["blocking", "json"] } rustpython = { git = "https://github.com/RustPython/RustPython.git", features = [ "threading", "stdlib", "stdio", + "freeze-stdlib", "importlib", "ssl", ], default-features = false } diff --git a/crates/yt_dlp/src/info_json.rs b/crates/yt_dlp/src/info_json.rs new file mode 100644 index 0000000..31f4a69 --- /dev/null +++ b/crates/yt_dlp/src/info_json.rs @@ -0,0 +1,60 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use rustpython::vm::{ + PyRef, VirtualMachine, + builtins::{PyDict, PyStr}, +}; + +pub type InfoJson = serde_json::Map<String, serde_json::Value>; + +pub fn json_loads( + input: serde_json::Map<String, serde_json::Value>, + vm: &VirtualMachine, +) -> PyRef<PyDict> { + let json = vm.import("json", 0).expect("Module exists"); + let loads = json.get_attr("loads", vm).expect("Method exists"); + let self_str = serde_json::to_string(&serde_json::Value::Object(input)).expect("Vaild json"); + let dict = loads + .call((self_str,), vm) + .expect("Vaild json is always a valid dict"); + + dict.downcast().expect("Should always be a dict") +} + +/// # Panics +/// If expectation about python operations fail. +pub fn json_dumps( + input: PyRef<PyDict>, + vm: &VirtualMachine, +) -> serde_json::Map<String, serde_json::Value> { + let json = vm.import("json", 0).expect("Module exists"); + let dumps = json.get_attr("dumps", vm).expect("Method exists"); + let dict = dumps + .call((input,), vm) + .map_err(|err| vm.print_exception(err)) + .expect("Might not always work, but for our dicts it works"); + + let string: PyRef<PyStr> = dict.downcast().expect("Should always be a string"); + + let real_string = string.to_str().expect("Should be valid utf8"); + + // { + // let mut file = File::create("debug.dump.json").unwrap(); + // write!(file, "{}", real_string).unwrap(); + // } + + let value: serde_json::Value = serde_json::from_str(real_string).expect("Should be valid json"); + + match value { + serde_json::Value::Object(map) => map, + _ => unreachable!("These should not be json.dumps output"), + } +} diff --git a/crates/yt_dlp/src/lib.rs b/crates/yt_dlp/src/lib.rs index e7b37c6..a1db606 100644 --- a/crates/yt_dlp/src/lib.rs +++ b/crates/yt_dlp/src/lib.rs @@ -10,27 +10,29 @@ //! The `yt_dlp` interface is completely contained in the [`YoutubeDL`] structure. -use std::{self, env, fmt::Display, path::PathBuf}; +use std::path::PathBuf; use indexmap::IndexMap; -use log::{Level, debug, error, info, log_enabled}; -use logging::setup_logging; -use post_processors::PostProcessor; -use rustpython::{ - InterpreterConfig, - vm::{ - self, AsObject, Interpreter, PyObjectRef, PyPayload, PyRef, VirtualMachine, - builtins::{PyBaseException, PyBaseExceptionRef, PyDict, PyList, PyStr}, - function::{FuncArgs, KwArgs, PosArgs}, - py_io::Write, - suggestion::offer_suggestions, - }, +use log::info; +use rustpython::vm::{ + Interpreter, PyObjectRef, PyRef, VirtualMachine, + builtins::{PyDict, PyList, PyStr}, + function::{FuncArgs, KwArgs, PosArgs}, }; use url::Url; -mod logging; +use crate::{ + info_json::{InfoJson, json_dumps, json_loads}, + python_error::PythonError, +}; + +pub mod info_json; +pub mod options; pub mod post_processors; pub mod progress_hook; +pub mod python_error; + +mod logging; #[macro_export] macro_rules! json_get { @@ -73,7 +75,6 @@ pub struct YoutubeDL { youtube_dl_class: PyObjectRef, yt_dlp_module: PyObjectRef, options: serde_json::Map<String, serde_json::Value>, - post_processors: Vec<Box<dyn PostProcessor>>, } impl std::fmt::Debug for YoutubeDL { @@ -84,148 +85,6 @@ impl std::fmt::Debug for YoutubeDL { } impl YoutubeDL { - /// Construct this instance from options. - /// - /// # Panics - /// If `yt_dlp` changed their interface. - /// - /// # Errors - /// If a python call fails. - pub fn from_options(options: YoutubeDLOptions) -> Result<Self, build::Error> { - let mut settings = vm::Settings::default(); - if let Ok(python_path) = env::var("PYTHONPATH") { - for path in python_path.split(':') { - settings.path_list.push(path.to_owned()); - } - } else { - error!( - "No PYTHONPATH found or invalid utf8. \ - This means, that you probably did not \ - supply the yt_dlp!" - ); - } - - settings.install_signal_handlers = false; - - // NOTE(@bpeetz): Another value leads to an internal codegen error. <2025-06-13> - settings.optimize = 0; - - settings.isolated = true; - - let interpreter = InterpreterConfig::new() - .init_stdlib() - .settings(settings) - .interpreter(); - - let output_options = options.options.clone(); - - let (yt_dlp_module, youtube_dl_class) = match interpreter.enter(|vm| { - let yt_dlp_module = vm.import("yt_dlp", 0)?; - let class = yt_dlp_module.get_attr("YoutubeDL", vm)?; - - let opts = json_loads(options.options, vm); - if let Some(function) = options.progress_hook { - opts.get_or_insert(vm, vm.new_pyobj("progress_hooks"), || { - let hook: PyObjectRef = vm.new_function("progress_hook", function).into(); - vm.new_pyobj(vec![hook]) - }) - .expect("Should work?"); - } - - { - // Unconditionally set a logger. - // Otherwise, yt_dlp will log to stderr. - - /// Is the specified record to be logged? Returns false for no, - /// true for yes. Filters can either modify log records in-place or - /// return a completely different record instance which will replace - /// the original log record in any future processing of the event. - fn filter_error_log(mut input: FuncArgs, vm: &VirtualMachine) -> bool { - let record = input.args.remove(0); - - // Filter out all error logs (they are propagated as rust errors) - let levelname: PyRef<PyStr> = record - .get_attr("levelname", vm) - .expect("This should exist") - .downcast() - .expect("This should be a String"); - - let return_value = levelname.as_str() != "ERROR"; - - if log_enabled!(Level::Debug) && !return_value { - let message: String = { - let get_message = record.get_attr("getMessage", vm).expect("Is set"); - let message: PyRef<PyStr> = get_message - .call((), vm) - .expect("Can be called") - .downcast() - .expect("Downcasting works"); - - message.as_str().to_owned() - }; - - debug!("Swollowed error message: '{message}'"); - } - return_value - } - - let logging = setup_logging(vm, "yt_dlp")?; - let ytdl_logger = { - let get_logger = logging.get_item("getLogger", vm)?; - get_logger.call(("yt_dlp",), vm)? - }; - - { - let args = FuncArgs::new( - PosArgs::new(vec![]), - KwArgs::new({ - let mut map = IndexMap::new(); - // Ensure that all events are logged by setting - // the log level to NOTSET (we filter on rust's side) - map.insert("level".to_owned(), vm.new_pyobj(0)); - map - }), - ); - - let basic_config = logging.get_item("basicConfig", vm)?; - basic_config.call(args, vm)?; - } - - { - let add_filter = ytdl_logger.get_attr("addFilter", vm)?; - add_filter.call( - (vm.new_function("yt_dlp_error_filter", filter_error_log),), - vm, - )?; - } - - opts.set_item("logger", ytdl_logger, vm)?; - } - - let youtube_dl_class = class.call((opts,), vm)?; - - Ok::<_, PyRef<PyBaseException>>((yt_dlp_module, youtube_dl_class)) - }) { - Ok(ok) => Ok(ok), - Err(err) => { - // TODO(@bpeetz): Do we want to run `interpreter.finalize` here? <2025-06-14> - // interpreter.finalize(Some(err)); - interpreter.enter(|vm| { - let buffer = process_exception(vm, &err); - Err(build::Error::Python(buffer)) - }) - } - }?; - - Ok(Self { - interpreter, - youtube_dl_class, - yt_dlp_module, - options: output_options, - post_processors: options.post_processors, - }) - } - /// # Panics /// /// If `yt_dlp` changed their location or type of `__version__`. @@ -413,43 +272,13 @@ impl YoutubeDL { let result = value.downcast::<PyDict>().expect("This should stay a dict"); - let mut json = json_dumps(result, vm); - - for pp in &self.post_processors { - if pp - .extractors() - .iter() - .any(|extractor| *extractor == json_get!(json, "extractor_key", as_str)) - { - json = pp.process(json)?; - } else { - error!("Extractor not found for {pp:#?}"); - } - } - - Ok(json) - } -} - -#[derive(thiserror::Error, Debug)] -pub struct PythonError(pub String); - -impl Display for PythonError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Python threw an exception: {}", self.0) - } -} - -impl PythonError { - fn from_exception(vm: &VirtualMachine, exc: &PyRef<PyBaseException>) -> Self { - let buffer = process_exception(vm, exc); - Self(buffer) + Ok(json_dumps(result, vm)) } } #[allow(missing_docs)] pub mod process_ie_result { - use crate::{PythonError, prepare}; + use crate::{prepare, python_error::PythonError}; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -462,7 +291,7 @@ pub mod process_ie_result { } #[allow(missing_docs)] pub mod extract_info { - use crate::{PythonError, prepare}; + use crate::{prepare, python_error::PythonError}; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -475,221 +304,11 @@ pub mod extract_info { } #[allow(missing_docs)] pub mod prepare { - use crate::{PythonError, post_processors}; + use crate::python_error::PythonError; #[derive(Debug, thiserror::Error)] pub enum Error { #[error(transparent)] Python(#[from] PythonError), - - #[error("Failed to run a post processor")] - PostProcessorRun(#[from] post_processors::Error), - } -} - -pub type InfoJson = serde_json::Map<String, serde_json::Value>; -pub type ProgressHookFunction = fn(input: FuncArgs, vm: &VirtualMachine); - -/// Options, that are used to customize the download behaviour. -/// -/// In the future, this might get a Builder api. -/// -/// See `help(yt_dlp.YoutubeDL())` from python for a full list of available options. -#[derive(Default, Debug)] -pub struct YoutubeDLOptions { - options: serde_json::Map<String, serde_json::Value>, - progress_hook: Option<ProgressHookFunction>, - post_processors: Vec<Box<dyn PostProcessor>>, -} - -impl YoutubeDLOptions { - #[must_use] - pub fn new() -> Self { - let me = Self { - options: serde_json::Map::new(), - progress_hook: None, - post_processors: vec![], - }; - - me.with_post_processor(post_processors::dearrow::DeArrowPP) - } - - #[must_use] - pub fn set(self, key: impl Into<String>, value: impl Into<serde_json::Value>) -> Self { - let mut options = self.options; - options.insert(key.into(), value.into()); - - Self { options, ..self } - } - - #[must_use] - pub fn with_progress_hook(self, progress_hook: ProgressHookFunction) -> Self { - if let Some(_previous_hook) = self.progress_hook { - todo!() - } else { - Self { - progress_hook: Some(progress_hook), - ..self - } - } - } - - #[must_use] - pub fn with_post_processor<P: PostProcessor + 'static>(mut self, post_processor: P) -> Self { - self.post_processors.push(Box::new(post_processor)); - self - } - - /// # Errors - /// If the underlying [`YoutubeDL::from_options`] errors. - pub fn build(self) -> Result<YoutubeDL, build::Error> { - YoutubeDL::from_options(self) - } - - #[must_use] - pub fn from_json_options(options: serde_json::Map<String, serde_json::Value>) -> Self { - Self { - options, - ..Self::new() - } - } - - #[must_use] - pub fn get(&self, key: &str) -> Option<&serde_json::Value> { - self.options.get(key) - } -} - -#[allow(missing_docs)] -pub mod build { - #[derive(Debug, thiserror::Error)] - pub enum Error { - #[error("Python threw an exception: {0}")] - Python(String), } } - -fn json_loads( - input: serde_json::Map<String, serde_json::Value>, - vm: &VirtualMachine, -) -> PyRef<PyDict> { - let json = vm.import("json", 0).expect("Module exists"); - let loads = json.get_attr("loads", vm).expect("Method exists"); - let self_str = serde_json::to_string(&serde_json::Value::Object(input)).expect("Vaild json"); - let dict = loads - .call((self_str,), vm) - .expect("Vaild json is always a valid dict"); - - dict.downcast().expect("Should always be a dict") -} - -/// # Panics -/// If expectation about python operations fail. -pub fn json_dumps( - input: PyRef<PyDict>, - vm: &VirtualMachine, -) -> serde_json::Map<String, serde_json::Value> { - let json = vm.import("json", 0).expect("Module exists"); - let dumps = json.get_attr("dumps", vm).expect("Method exists"); - let dict = dumps - .call((input,), vm) - .map_err(|err| vm.print_exception(err)) - .expect("Might not always work, but for our dicts it works"); - - let string: PyRef<PyStr> = dict.downcast().expect("Should always be a string"); - - let real_string = string.to_str().expect("Should be valid utf8"); - - // { - // let mut file = File::create("debug.dump.json").unwrap(); - // write!(file, "{}", real_string).unwrap(); - // } - - let value: serde_json::Value = serde_json::from_str(real_string).expect("Should be valid json"); - - match value { - serde_json::Value::Object(map) => map, - _ => unreachable!("These should not be json.dumps output"), - } -} - -// Inlined and changed from `vm.write_exception_inner` -fn write_exception<W: Write>( - vm: &VirtualMachine, - output: &mut W, - exc: &PyBaseExceptionRef, -) -> Result<(), W::Error> { - let varargs = exc.args(); - let args_repr = { - match varargs.len() { - 0 => vec![], - 1 => { - let args0_repr = if true { - varargs[0] - .str(vm) - .unwrap_or_else(|_| PyStr::from("<element str() failed>").into_ref(&vm.ctx)) - } else { - varargs[0].repr(vm).unwrap_or_else(|_| { - PyStr::from("<element repr() failed>").into_ref(&vm.ctx) - }) - }; - vec![args0_repr] - } - _ => varargs - .iter() - .map(|vararg| { - vararg.repr(vm).unwrap_or_else(|_| { - PyStr::from("<element repr() failed>").into_ref(&vm.ctx) - }) - }) - .collect(), - } - }; - - let exc_class = exc.class(); - - if exc_class.fast_issubclass(vm.ctx.exceptions.syntax_error) { - unreachable!( - "A syntax error should never be raised, \ - as yt_dlp should not have them and neither our embedded code" - ); - } - - let exc_name = exc_class.name(); - match args_repr.len() { - 0 => write!(output, "{exc_name}"), - 1 => write!(output, "{}: {}", exc_name, args_repr[0]), - _ => write!( - output, - "{}: ({})", - exc_name, - args_repr - .iter() - .map(|val| val.as_str()) - .collect::<Vec<_>>() - .join(", "), - ), - }?; - - match offer_suggestions(exc, vm) { - Some(suggestions) => { - write!(output, ". Did you mean: '{suggestions}'?") - } - None => Ok(()), - } -} - -fn process_exception(vm: &VirtualMachine, err: &PyBaseExceptionRef) -> String { - let mut buffer = String::new(); - write_exception(vm, &mut buffer, err) - .expect("We are writing into an *in-memory* string, it will always work"); - - if log_enabled!(Level::Debug) { - let mut output = String::new(); - vm.write_exception(&mut output, err) - .expect("We are writing into an *in-memory* string, it will always work"); - debug!("Python threw an exception: {output}"); - } - - buffer -} diff --git a/crates/yt_dlp/src/logging.rs b/crates/yt_dlp/src/logging.rs index 5cb4c1d..112836e 100644 --- a/crates/yt_dlp/src/logging.rs +++ b/crates/yt_dlp/src/logging.rs @@ -15,7 +15,7 @@ use log::{Level, MetadataBuilder, Record, logger}; use rustpython::vm::{ PyObjectRef, PyRef, PyResult, VirtualMachine, - builtins::{PyInt, PyList, PyStr}, + builtins::{PyInt, PyStr}, convert::ToPyObject, function::FuncArgs, }; @@ -167,31 +167,5 @@ def basicConfig(*pargs, **kwargs): "<embedded logging inintializing code>".to_owned(), )?; - let all: PyRef<PyList> = logging - .get_attr("__all__", vm)? - .downcast() - .expect("Is a list"); - all.borrow_vec_mut().push(vm.new_pyobj("HostHandler")); - - // { - // let logging_dict = logging.dict().expect("Exists"); - // - // for (key, val) in scope.globals { - // let key: PyRef<PyStr> = key.downcast().expect("Is a string"); - // - // if !logging_dict.contains_key(key.as_str(), vm) { - // logging_dict.set_item(key.as_str(), val, vm)?; - // } - // } - // - // for (key, val) in scope.locals { - // let key: PyRef<PyStr> = key.downcast().expect("Is a string"); - // - // if !logging_dict.contains_key(key.as_str(), vm) { - // logging_dict.set_item(key.as_str(), val, vm)?; - // } - // } - // } - Ok(scope.globals.to_pyobject(vm)) } diff --git a/crates/yt_dlp/src/options.rs b/crates/yt_dlp/src/options.rs new file mode 100644 index 0000000..182b8a1 --- /dev/null +++ b/crates/yt_dlp/src/options.rs @@ -0,0 +1,280 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::env; + +use indexmap::IndexMap; +use log::{Level, debug, error, log_enabled}; +use rustpython::{ + InterpreterConfig, + vm::{ + self, PyObjectRef, PyRef, PyResult, VirtualMachine, + builtins::{PyBaseException, PyStr}, + function::{FuncArgs, KwArgs, PosArgs}, + }, +}; + +use crate::{ + YoutubeDL, json_loads, logging::setup_logging, post_processors, python_error::process_exception, +}; + +/// Wrap your function with [`mk_python_function`]. +pub type ProgressHookFunction = fn(input: FuncArgs, vm: &VirtualMachine); + +pub type PostProcessorFunction = fn(vm: &VirtualMachine) -> PyResult<PyObjectRef>; + +/// Options, that are used to customize the download behaviour. +/// +/// In the future, this might get a Builder api. +/// +/// See `help(yt_dlp.YoutubeDL())` from python for a full list of available options. +#[derive(Default, Debug)] +pub struct YoutubeDLOptions { + options: serde_json::Map<String, serde_json::Value>, + progress_hook: Option<ProgressHookFunction>, + post_processors: Vec<PostProcessorFunction>, +} + +impl YoutubeDLOptions { + #[must_use] + pub fn new() -> Self { + let me = Self { + options: serde_json::Map::new(), + progress_hook: None, + post_processors: vec![], + }; + + me.with_post_processor(post_processors::dearrow::process) + } + + #[must_use] + pub fn set(self, key: impl Into<String>, value: impl Into<serde_json::Value>) -> Self { + let mut options = self.options; + options.insert(key.into(), value.into()); + + Self { options, ..self } + } + + #[must_use] + pub fn with_progress_hook(self, progress_hook: ProgressHookFunction) -> Self { + if let Some(_previous_hook) = self.progress_hook { + todo!() + } else { + Self { + progress_hook: Some(progress_hook), + ..self + } + } + } + + #[must_use] + pub fn with_post_processor(mut self, pp: PostProcessorFunction) -> Self { + self.post_processors.push(pp); + self + } + + /// # Errors + /// If the underlying [`YoutubeDL::from_options`] errors. + pub fn build(self) -> Result<YoutubeDL, build::Error> { + YoutubeDL::from_options(self) + } + + #[must_use] + pub fn from_json_options(options: serde_json::Map<String, serde_json::Value>) -> Self { + Self { + options, + ..Self::new() + } + } + + #[must_use] + pub fn get(&self, key: &str) -> Option<&serde_json::Value> { + self.options.get(key) + } +} + +impl YoutubeDL { + /// Construct this instance from options. + /// + /// # Panics + /// If `yt_dlp` changed their interface. + /// + /// # Errors + /// If a python call fails. + #[allow(clippy::too_many_lines)] + pub fn from_options(options: YoutubeDLOptions) -> Result<Self, build::Error> { + let mut settings = vm::Settings::default(); + if let Ok(python_path) = env::var("PYTHONPATH") { + for path in python_path.split(':') { + settings.path_list.push(path.to_owned()); + } + } else { + error!( + "No PYTHONPATH found or invalid utf8. \ + This means, that you probably did not \ + supply a yt_dlp python package!" + ); + } + + settings.install_signal_handlers = false; + + // NOTE(@bpeetz): Another value leads to an internal codegen error. <2025-06-13> + settings.optimize = 0; + + settings.isolated = true; + + let interpreter = InterpreterConfig::new() + .init_stdlib() + .settings(settings) + .interpreter(); + + let output_options = options.options.clone(); + + let (yt_dlp_module, youtube_dl_class) = match interpreter.enter(|vm| { + let yt_dlp_module = vm.import("yt_dlp", 0)?; + let class = yt_dlp_module.get_attr("YoutubeDL", vm)?; + + let opts = json_loads(options.options, vm); + + { + // Setup the progress hook + if let Some(function) = options.progress_hook { + opts.get_or_insert(vm, vm.new_pyobj("progress_hooks"), || { + let hook: PyObjectRef = vm.new_function("progress_hook", function).into(); + vm.new_pyobj(vec![hook]) + }) + .expect("Should work?"); + } + } + + { + // Unconditionally set a logger. + // Otherwise, yt_dlp will log to stderr. + + /// Is the specified record to be logged? Returns false for no, + /// true for yes. Filters can either modify log records in-place or + /// return a completely different record instance which will replace + /// the original log record in any future processing of the event. + fn filter_error_log(mut input: FuncArgs, vm: &VirtualMachine) -> bool { + let record = input.args.remove(0); + + // Filter out all error logs (they are propagated as rust errors) + let levelname: PyRef<PyStr> = record + .get_attr("levelname", vm) + .expect("This should exist") + .downcast() + .expect("This should be a String"); + + let return_value = levelname.as_str() != "ERROR"; + + if log_enabled!(Level::Debug) && !return_value { + let message: String = { + let get_message = record.get_attr("getMessage", vm).expect("Is set"); + let message: PyRef<PyStr> = get_message + .call((), vm) + .expect("Can be called") + .downcast() + .expect("Downcasting works"); + + message.as_str().to_owned() + }; + + debug!("Swollowed error message: '{message}'"); + } + return_value + } + + let logging = setup_logging(vm, "yt_dlp")?; + let ytdl_logger = { + let get_logger = logging.get_item("getLogger", vm)?; + get_logger.call(("yt_dlp",), vm)? + }; + + { + let args = FuncArgs::new( + PosArgs::new(vec![]), + KwArgs::new({ + let mut map = IndexMap::new(); + // Ensure that all events are logged by setting + // the log level to NOTSET (we filter on rust's side) + map.insert("level".to_owned(), vm.new_pyobj(0)); + map + }), + ); + + let basic_config = logging.get_item("basicConfig", vm)?; + basic_config.call(args, vm)?; + } + + { + let add_filter = ytdl_logger.get_attr("addFilter", vm)?; + add_filter.call( + (vm.new_function("yt_dlp_error_filter", filter_error_log),), + vm, + )?; + } + + opts.set_item("logger", ytdl_logger, vm)?; + } + + let youtube_dl_class = class.call((opts,), vm)?; + + { + // Setup the post processors + + let add_post_processor_fun = youtube_dl_class.get_attr("add_post_processor", vm)?; + + for pp in options.post_processors { + let args = { + FuncArgs::new( + PosArgs::new(vec![pp(vm)?]), + KwArgs::new({ + let mut map = IndexMap::new(); + // "when" can take any value in yt_dlp.utils.POSTPROCESS_WHEN + map.insert("when".to_owned(), vm.new_pyobj("pre_process")); + map + }), + ) + }; + + add_post_processor_fun.call(args, vm)?; + } + } + + Ok::<_, PyRef<PyBaseException>>((yt_dlp_module, youtube_dl_class)) + }) { + Ok(ok) => Ok(ok), + Err(err) => { + // TODO(@bpeetz): Do we want to run `interpreter.finalize` here? <2025-06-14> + // interpreter.finalize(Some(err)); + interpreter.enter(|vm| { + let buffer = process_exception(vm, &err); + Err(build::Error::Python(buffer)) + }) + } + }?; + + Ok(Self { + interpreter, + youtube_dl_class, + yt_dlp_module, + options: output_options, + }) + } +} + +#[allow(missing_docs)] +pub mod build { + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error("Python threw an exception: {0}")] + Python(String), + } +} diff --git a/crates/yt_dlp/src/post_processors/dearrow.rs b/crates/yt_dlp/src/post_processors/dearrow.rs index bdbea7c..ab5478b 100644 --- a/crates/yt_dlp/src/post_processors/dearrow.rs +++ b/crates/yt_dlp/src/post_processors/dearrow.rs @@ -8,60 +8,118 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use log::{info, warn}; +use curl::easy::Easy; +use log::{error, info, warn}; +use rustpython::vm::{ + PyRef, VirtualMachine, + builtins::{PyDict, PyStr}, +}; use serde::{Deserialize, Serialize}; -use crate::{InfoJson, json_get}; +use crate::{pydict_cast, pydict_get, wrap_post_processor}; -use super::PostProcessor; +wrap_post_processor!("DeArrow", unwrapped_process, process); -#[derive(Debug, Clone, Copy)] -pub struct DeArrowPP; - -impl PostProcessor for DeArrowPP { - fn extractors(&self) -> &'static [&'static str] { - &["Youtube"] +/// # Errors +/// If the API access fails. +pub fn unwrapped_process(info: PyRef<PyDict>, vm: &VirtualMachine) -> Result<PyRef<PyDict>, Error> { + if pydict_get!(@vm, info, "extractor_key", PyStr).as_str() != "Youtube" { + warn!("DeArrow: Extractor did not match, exiting."); + return Ok(info); } - fn process(&self, mut info: InfoJson) -> Result<InfoJson, super::Error> { - let mut output: DeArrowApi = reqwest::blocking::get(format!( - "https://sponsor.ajay.app/api/branding?videoID={}", - json_get!(info, "id", as_str) - ))? - .json()?; - - output.titles.reverse(); - - let title_len = output.titles.len(); - loop { - let Some(title) = output.titles.pop() else { - break; - }; - - if (title.locked || title.votes < 1) && title_len > 1 { - info!( - "Skipping title {:#?}, as it is not good enough", - title.value - ); - // Skip titles that are not “good” enough. - continue; - } - - if let Some(old_title) = info.insert( - "title".to_owned(), - serde_json::Value::String(title.value.clone()), - ) { - warn!("Updating title from {:#?} to {:#?}", old_title, title.value); - info.insert("original_title".to_owned(), old_title); - } else { - warn!("Setting title to {:#?}", title.value); - } - - break; + let mut output: DeArrowApi = { + let output_bytes = { + let mut dst = Vec::new(); + + let mut easy = Easy::new(); + easy.url( + format!( + "https://sponsor.ajay.app/api/branding?videoID={}", + pydict_get!(@vm, info, "id", PyStr).as_str() + ) + .as_str(), + )?; + + let mut transfer = easy.transfer(); + transfer.write_function(|data| { + dst.extend_from_slice(data); + Ok(data.len()) + })?; + transfer.perform()?; + drop(transfer); + + dst + }; + + serde_json::from_slice(&output_bytes)? + }; + + // We pop the titles, so we need this vector reversed. + output.titles.reverse(); + + let title_len = output.titles.len(); + let selected = loop { + let Some(title) = output.titles.pop() else { + break false; + }; + + if (title.locked || title.votes < 1) && title_len > 1 { + info!( + "DeArrow: Skipping title {:#?}, as it is not good enough", + title.value + ); + // Skip titles that are not “good” enough. + continue; } - Ok(info) + update_title(&info, &title.value, vm); + + break true; + }; + + if !selected && title_len != 0 { + // No title was selected, even though we had some titles. + // Just pick the first one in this case. + update_title(&info, &output.titles[0].value, vm); } + + Ok(info) +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Failed to access the DeArrow api: {0}")] + Get(#[from] curl::Error), + + #[error("Failed to deserialize a api json return object: {0}")] + Deserialize(#[from] serde_json::Error), +} + +fn update_title(info: &PyRef<PyDict>, new_title: &str, vm: &VirtualMachine) { + assert!(!info.contains_key("original_title", vm)); + + if let Ok(old_title) = info.get_item("title", vm) { + warn!( + "DeArrow: Updating title from {:#?} to {:#?}", + pydict_cast!(@ref old_title, PyStr).as_str(), + new_title + ); + + info.set_item("original_title", old_title, vm) + .expect("We checked, it is a new key"); + } else { + warn!("DeArrow: Setting title to {new_title:#?}"); + } + + let cleaned_title = { + // NOTE(@bpeetz): DeArrow uses `>` as a “Don't format the next word” mark. + // They should be removed, if one does not use a auto-formatter. <2025-06-16> + new_title.replace('>', "") + }; + + info.set_item("title", vm.new_pyobj(cleaned_title), vm) + .expect("This should work?"); } #[derive(Serialize, Deserialize)] @@ -77,7 +135,14 @@ struct DeArrowApi { video_duration: Option<f64>, #[serde(alias = "casualVotes")] - casual_votes: Vec<String>, + casual_votes: Vec<CasualVote>, +} + +#[derive(Serialize, Deserialize)] +struct CasualVote { + id: String, + count: u32, + title: String, } #[derive(Serialize, Deserialize)] diff --git a/crates/yt_dlp/src/post_processors/mod.rs b/crates/yt_dlp/src/post_processors/mod.rs index 65801c2..00b0ad5 100644 --- a/crates/yt_dlp/src/post_processors/mod.rs +++ b/crates/yt_dlp/src/post_processors/mod.rs @@ -8,23 +8,116 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use crate::InfoJson; - pub mod dearrow; -pub trait PostProcessor: std::fmt::Debug + Send { - /// Process a [`InfoJson`] object and return the updated one. - /// - /// # Errors - /// If the processing steps failed. - fn process(&self, info: InfoJson) -> Result<InfoJson, Error>; +#[macro_export] +macro_rules! pydict_get { + (@$vm:expr, $value:expr, $name:literal, $into:ident) => {{ + match $value.get_item($name, $vm) { + Ok(val) => $crate::pydict_cast!(val, $into), + Err(_) => panic!( + concat!( + "Expected '", + $name, + "' to be a key for the'", + stringify!($value), + "' py dictionary: {:#?}" + ), + $value + ), + } + }}; +} - /// The supported extractors for this post processor - fn extractors(&self) -> &'static [&'static str]; +#[macro_export] +macro_rules! pydict_cast { + ($value:expr, $into:ident) => {{ + match $value.downcast::<$into>() { + Ok(result) => result, + Err(val) => panic!( + concat!( + "Expected to be able to downcast value ({:#?}) as ", + stringify!($into) + ), + val + ), + } + }}; + (@ref $value:expr, $into:ident) => {{ + match $value.downcast_ref::<$into>() { + Some(result) => result, + None => panic!( + concat!( + "Expected to be able to downcast value ({:#?}) as ", + stringify!($into) + ), + $value + ), + } + }}; } -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("Failed to access a api: {0}")] - Get(#[from] reqwest::Error), +#[macro_export] +macro_rules! wrap_post_processor { + ($name:literal, $unwrap:ident, $wrapped:ident) => { + use $crate::progress_hook::__priv::vm; + + /// # Errors + /// - If the underlying function returns an error. + /// - If python operations fail. + pub fn $wrapped(vm: &vm::VirtualMachine) -> vm::PyResult<vm::PyObjectRef> { + fn actual_processor( + mut input: vm::function::FuncArgs, + vm: &vm::VirtualMachine, + ) -> vm::PyResult<vm::PyRef<vm::builtins::PyDict>> { + let input = input + .args + .remove(0) + .downcast::<vm::builtins::PyDict>() + .expect("Should be a py dict"); + + let output = match unwrapped_process(input, vm) { + Ok(ok) => ok, + Err(err) => { + return Err(vm.new_runtime_error(err.to_string())); + } + }; + + Ok(output) + } + + let scope = vm.new_scope_with_builtins(); + + scope.globals.set_item( + "actual_processor", + vm.new_function("actual_processor", actual_processor).into(), + vm, + )?; + + let local_scope = scope.clone(); + vm.run_code_string( + local_scope, + format!( + " +import yt_dlp + +class {}(yt_dlp.postprocessor.PostProcessor): + def run(self, info): + info = actual_processor(info) + return [], info + +inst = {}() +", + $name, $name + ) + .as_str(), + "<embedded post processor initializing code>".to_owned(), + )?; + + Ok(scope + .globals + .get_item("inst", vm) + .expect("We just declared it")) + } + }; } diff --git a/crates/yt_dlp/src/progress_hook.rs b/crates/yt_dlp/src/progress_hook.rs index 43f85e0..b42ae21 100644 --- a/crates/yt_dlp/src/progress_hook.rs +++ b/crates/yt_dlp/src/progress_hook.rs @@ -12,21 +12,21 @@ macro_rules! mk_python_function { ($name:ident, $new_name:ident) => { pub fn $new_name( - mut args: $crate::progress_hook::rustpython::vm::function::FuncArgs, - vm: &$crate::progress_hook::rustpython::vm::VirtualMachine, + mut args: $crate::progress_hook::__priv::vm::function::FuncArgs, + vm: &$crate::progress_hook::__priv::vm::VirtualMachine, ) { - use $crate::progress_hook::rustpython; + use $crate::progress_hook::__priv::vm; let input = { - let dict: rustpython::vm::PyRef<rustpython::vm::builtins::PyDict> = args + let dict: vm::PyRef<vm::builtins::PyDict> = args .args .remove(0) .downcast() .expect("The progress hook is always called with these args"); - let new_dict = rustpython::vm::builtins::PyDict::new_ref(&vm.ctx); + let new_dict = vm::builtins::PyDict::new_ref(&vm.ctx); dict.into_iter() .filter_map(|(name, value)| { - let real_name: rustpython::vm::PyRefExact<rustpython::vm::builtins::PyStr> = + let real_name: vm::PyRefExact<vm::builtins::PyStr> = name.downcast_exact(vm).expect("Is a string"); let name_str = real_name.to_str().expect("Is a string"); if name_str.starts_with('_') { @@ -41,11 +41,14 @@ macro_rules! mk_python_function { .expect("This is a transpositions, should always be valid"); }); - $crate::json_dumps(new_dict, vm) + $crate::progress_hook::__priv::json_dumps(new_dict, vm) }; $name(input).expect("Shall not fail!"); } }; } -pub use rustpython; +pub mod __priv { + pub use crate::info_json::{json_dumps, json_loads}; + pub use rustpython::vm; +} diff --git a/crates/yt_dlp/src/python_error.rs b/crates/yt_dlp/src/python_error.rs new file mode 100644 index 0000000..9513956 --- /dev/null +++ b/crates/yt_dlp/src/python_error.rs @@ -0,0 +1,116 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::Display; + +use log::{Level, debug, log_enabled}; +use rustpython::vm::{ + AsObject, PyPayload, PyRef, VirtualMachine, + builtins::{PyBaseException, PyBaseExceptionRef, PyStr}, + py_io::Write, + suggestion::offer_suggestions, +}; + +#[derive(thiserror::Error, Debug)] +pub struct PythonError(pub String); + +impl Display for PythonError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Python threw an exception: {}", self.0) + } +} + +impl PythonError { + pub(super) fn from_exception(vm: &VirtualMachine, exc: &PyRef<PyBaseException>) -> Self { + let buffer = process_exception(vm, exc); + Self(buffer) + } +} + +pub(super) fn process_exception(vm: &VirtualMachine, err: &PyBaseExceptionRef) -> String { + let mut buffer = String::new(); + write_exception(vm, &mut buffer, err) + .expect("We are writing into an *in-memory* string, it will always work"); + + if log_enabled!(Level::Debug) { + let mut output = String::new(); + vm.write_exception(&mut output, err) + .expect("We are writing into an *in-memory* string, it will always work"); + debug!("Python threw an exception: {output}"); + } + + buffer +} + +// Inlined and changed from `vm.write_exception_inner` +fn write_exception<W: Write>( + vm: &VirtualMachine, + output: &mut W, + exc: &PyBaseExceptionRef, +) -> Result<(), W::Error> { + let varargs = exc.args(); + let args_repr = { + match varargs.len() { + 0 => vec![], + 1 => { + let args0_repr = if true { + varargs[0] + .str(vm) + .unwrap_or_else(|_| PyStr::from("<element str() failed>").into_ref(&vm.ctx)) + } else { + varargs[0].repr(vm).unwrap_or_else(|_| { + PyStr::from("<element repr() failed>").into_ref(&vm.ctx) + }) + }; + vec![args0_repr] + } + _ => varargs + .iter() + .map(|vararg| { + vararg.repr(vm).unwrap_or_else(|_| { + PyStr::from("<element repr() failed>").into_ref(&vm.ctx) + }) + }) + .collect(), + } + }; + + let exc_class = exc.class(); + + if exc_class.fast_issubclass(vm.ctx.exceptions.syntax_error) { + unreachable!( + "A syntax error should never be raised, \ + as yt_dlp should not have them and neither our embedded code" + ); + } + + let exc_name = exc_class.name(); + match args_repr.len() { + 0 => write!(output, "{exc_name}"), + 1 => write!(output, "{}: {}", exc_name, args_repr[0]), + _ => write!( + output, + "{}: ({})", + exc_name, + args_repr + .iter() + .map(|val| val.as_str()) + .collect::<Vec<_>>() + .join(", "), + ), + }?; + + match offer_suggestions(exc, vm) { + Some(suggestions) => { + write!(output, ". Did you mean: '{suggestions}'?") + } + None => Ok(()), + } +} diff --git a/flake.lock b/flake.lock index fcf9d4c..b687a79 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1750005889, - "narHash": "sha256-5Ja4RfAWUqzX1B1MC/mSQzNBsTtXmlW4RQyPqmHVU90=", + "lastModified": 1750093821, + "narHash": "sha256-rumKjLR6VSoDG8eCiCLmwbuWDI+JnDzxaEWVF7F95OU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0fbc85d348db795d46453097b151c08712b86a84", + "rev": "526945c5798687e32d4a6f8a93660fe2ca152ae2", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1a6b43b..c90fca4 100644 --- a/flake.nix +++ b/flake.nix @@ -43,7 +43,7 @@ SDL2 ]; - yt = pkgs.callPackage ./package/package.nix {inherit tree-sitter-yts;}; + yt = pkgs.callPackage ./nix/package.nix {inherit tree-sitter-yts;}; tree-sitter-yts = pkgs.callPackage ./tree-sitter-yts/package.nix {}; treefmtEval = import ./treefmt.nix {inherit treefmt-nix pkgs;}; diff --git a/package/package.nix b/nix/package.nix index 5a29fad..f4d6eac 100644 --- a/package/package.nix +++ b/nix/package.nix @@ -14,6 +14,7 @@ # buildInputs mpv-unwrapped, python3Packages, + python3, ffmpeg, openssl, libffi, @@ -61,7 +62,6 @@ rustPlatform.buildRustPackage (finalAttrs: { ]; buildInputs = [ - python3Packages.yt-dlp mpv-unwrapped.dev ffmpeg openssl @@ -110,7 +110,26 @@ rustPlatform.buildRustPackage (finalAttrs: { }; }; - postInstall = '' + postInstall = let + collectDeps = pkg: let + next = pkg.propagatedBuildInputs or []; + in + [pkg] + ++ next + ++ (lib.flatten (builtins.map collectDeps next)); + + loadPythonDep = der: "${der}/lib/python${lib.versions.majorMinor python3.version}/site-packages"; + + pythonPath = builtins.concatStringsSep ":" (lib.lists.unique ( + builtins.map loadPythonDep ( + (collectDeps python3Packages.yt-dlp) + ++ [ + # HACK(@bpeetz): These packages are not picked up in the traversal up top. <2025-06-16> + python3Packages.chardet + ] + ) + )); + in '' installShellCompletion --cmd yt \ --bash <(COMPLETE=bash $out/bin/yt) \ --fish <(COMPLETE=fish $out/bin/yt) \ @@ -119,6 +138,7 @@ rustPlatform.buildRustPackage (finalAttrs: { # NOTE: We cannot clear the path, because we need access to the $EDITOR. <2025-04-04> wrapProgram $out/bin/yt \ --prefix PATH : ${lib.makeBinPath finalAttrs.buildInputs} \ - --set YTDLP_NO_PLUGINS 1 + --set YTDLP_NO_PLUGINS 1 \ + --set PYTHONPATH ${pythonPath} ''; }) |