diff options
Diffstat (limited to '')
153 files changed, 11759 insertions, 2434 deletions
diff --git a/crates/bytes/.gitignore b/contrib/external_commands_script.sh index 8876ea6..219eae7 100644..100755 --- a/crates/bytes/.gitignore +++ b/contrib/external_commands_script.sh @@ -1,6 +1,8 @@ +#! /usr/bin/env sh + # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> # SPDX-License-Identifier: GPL-3.0-or-later # # This file is part of Yt. @@ -8,4 +10,10 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -/target +riverctl focus-output next + +alacritty --title "floating please" --command "$@" + +riverctl focus-output next + +# vim: ft=sh diff --git a/crates/bytes/Cargo.lock b/crates/bytes/Cargo.lock deleted file mode 100644 index b30ba3d..0000000 --- a/crates/bytes/Cargo.lock +++ /dev/null @@ -1,65 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "bytes" -version = "1.0.0" -dependencies = [ - "serde", -] - -[[package]] -name = "proc-macro2" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "serde" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "syn" -version = "2.0.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" diff --git a/crates/bytes/src/serde.rs b/crates/bytes/src/serde.rs deleted file mode 100644 index 4341e32..0000000 --- a/crates/bytes/src/serde.rs +++ /dev/null @@ -1,19 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use serde::{Serialize, Serializer}; - -use crate::Bytes; - -impl Serialize for Bytes { - fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { - serializer.serialize_str(self.to_string().as_str()) - } -} diff --git a/crates/bytes/Cargo.toml b/crates/colors/Cargo.toml index 4439aa8..4edefcf 100644 --- a/crates/bytes/Cargo.toml +++ b/crates/colors/Cargo.toml @@ -1,7 +1,8 @@ # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 uutils developers +# SPDX-License-Identifier: MIT # # This file is part of Yt. # @@ -9,25 +10,17 @@ # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. [package] -name = "bytes" -description = "Simple byte formatting utilities" -keywords = [] -categories = [] -version.workspace = true -edition.workspace = true +name = "colors" authors.workspace = true license.workspace = true +description = "A owo-colors inspired color crate." +version.workspace = true +edition.workspace = true repository.workspace = true rust-version.workspace = true publish = false [dependencies] -serde.workspace = true - -[dev-dependencies] [lints] workspace = true - -[package.metadata.docs.rs] -all-features = true diff --git a/crates/colors/src/custom.rs b/crates/colors/src/custom.rs new file mode 100644 index 0000000..fd6b7b3 --- /dev/null +++ b/crates/colors/src/custom.rs @@ -0,0 +1,75 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// Taken from <https://github.com/owo-colors/owo-colors/blob/61f8bba2f5f80e9f4fa600fbfdf2c21656f1d523/src/colors/custom.rs> +// at 2025-07-16T18:05:55 CEST. + +const U8_TO_STR: [[u8; 3]; 256] = generate_lookup(); + +const fn generate_lookup() -> [[u8; 3]; 256] { + let mut table = [[0, 0, 0]; 256]; + + let mut i = 0; + while i < 256 { + table[i] = [ + b'0' + (i / 100) as u8, + b'0' + (i / 10 % 10) as u8, + b'0' + (i % 10) as u8, + ]; + i += 1; + } + + table +} + +#[derive(Clone, Copy)] +pub(crate) enum Plane { + Fg, + Bg, +} + +pub(crate) const fn rgb_to_ansi(r: u8, g: u8, b: u8, plane: Plane) -> [u8; 18] { + let mut buf = *b"\x1b[p8;2;rrr;ggg;bbb"; + + let r = U8_TO_STR[r as usize]; + let g = U8_TO_STR[g as usize]; + let b = U8_TO_STR[b as usize]; + + // p 2 + buf[2] = match plane { + Plane::Fg => b'3', + Plane::Bg => b'4', + }; + + // r 7 + buf[7] = r[0]; + buf[8] = r[1]; + buf[9] = r[2]; + + // g 11 + buf[11] = g[0]; + buf[12] = g[1]; + buf[13] = g[2]; + + // b 15 + buf[15] = b[0]; + buf[16] = b[1]; + buf[17] = b[2]; + + buf +} + +/// This exists since [`unwrap()`] isn't const-safe (it invokes formatting infrastructure) +pub(crate) const fn bytes_to_str(bytes: &'static [u8]) -> &'static str { + match core::str::from_utf8(bytes) { + Ok(o) => o, + Err(_e) => panic!("Const parsing &[u8] to a string failed!"), + } +} diff --git a/crates/colors/src/lib.rs b/crates/colors/src/lib.rs new file mode 100644 index 0000000..663e19a --- /dev/null +++ b/crates/colors/src/lib.rs @@ -0,0 +1,97 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{Display, Write}; + +use crate::{ + list::{elements, methods}, + support::{CSE, CSI, elements_inner}, +}; + +pub(crate) mod custom; +mod list; +mod support; + +#[derive(Debug)] +pub struct Canvas<I: Display>(I); + +impl<I: Display> Colorize for Canvas<I> { + fn render_into(self, base: &mut String, use_colors: bool) { + write!(base, "{}", self.0).expect("Is written into a string"); + + if use_colors { + // Reset the color and style, if we used colours. + base.write_str(CSI).expect("In-memory write"); + base.write_str("0").expect("In-memory write"); + base.write_str(CSE).expect("In-memory write"); + } + } +} + +pub trait IntoCanvas: Display + Sized { + fn into_canvas(self) -> Canvas<Self> { + Canvas(self) + } + + methods! { IntoCanvas } +} + +impl<I: Display> IntoCanvas for I {} + +pub trait Colorize: Sized { + /// Turn this colorized struct into a string, by writing into the base. + fn render_into(self, base: &mut String, use_colors: bool); + + /// Turn this colorized struct into a string for consumption. + fn render(self, use_colors: bool) -> String { + let mut base = String::new(); + self.render_into(&mut base, use_colors); + base + } + + methods! { Colorize } +} + +elements! {} + +#[cfg(test)] +mod tests { + use crate::{Colorize, IntoCanvas}; + + #[test] + fn test_colorize_basic() { + let base = "Base".green().render(true); + #[rustfmt::skip] + let expected = concat!( + "\x1b[32m", + "Base", + "\x1b[0m", + ); + + assert_eq!(base.as_str(), expected); + } + + #[test] + fn test_colorize_combo() { + let base = "Base".green().on_red().bold().strike_through().render(true); + + #[rustfmt::skip] + let expected = concat!( + "\x1b[9m", // strike_through + "\x1b[1m", // bold + "\x1b[41m", // on_red + "\x1b[32m", // green + "Base", + "\x1b[0m", + ); + + assert_eq!(base.as_str(), expected); + } +} diff --git a/crates/colors/src/list.rs b/crates/colors/src/list.rs new file mode 100644 index 0000000..35fcb83 --- /dev/null +++ b/crates/colors/src/list.rs @@ -0,0 +1,233 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::support::prepend_input; + +prepend_input! { + crate::support::methods_inner as methods (($tt:tt) -> {$tt}), + crate::support::elements_inner as elements, + + <shared_input> + { + // Colors + Black black 30, + OnBlack on_black 40, + + Red red 31, + OnRed on_red 41, + + Green green 32, + OnGreen on_green 42, + + Yellow yellow 33, + OnYellow on_yellow 43, + + Blue blue 34, + OnBlue on_blue 44, + + Magenta magenta 35, + OnMagenta on_magenta 45, + + Cyan cyan 36, + OnCyan on_cyan 46, + + White white 37, + OnWhite on_white 47, + + Default default 39, + OnDefault on_default 49, + + // Bright bright colors + BrightBlack bright_black 90, + OnBrightBlack on_bright_black 100, + + BrightRed bright_red 91, + OnBrightRed on_bright_red 101, + + BrightGreen bright_green 92, + OnBrightGreen on_bright_green 102, + + BrightYellow bright_yellow 93, + OnBrightYellow on_bright_yellow 103, + + BrightBlue bright_blue 94, + OnBrightBlue on_bright_blue 104, + + BrightMagenta bright_magenta 95, + OnBrightMagenta on_bright_magenta 105, + + BrightCyan bright_cyan 96, + OnBrightCyan on_bright_cyan 106, + + BrightWhite bright_white 97, + OnBrightWhite on_bright_white 107, + + // CSS colors + // TODO(@bpeetz): Also support background colors with these values. <2025-07-16> + AliceBlue alice_blue (240, 248, 255), + AntiqueWhite antique_white (250, 235, 215), + Aqua aqua (0, 255, 255), + Aquamarine aquamarine (127, 255, 212), + Azure azure (240, 255, 255), + Beige beige (245, 245, 220), + Bisque bisque (255, 228, 196), + // Black black (0, 0, 0), + BlanchedAlmond blanched_almond (255, 235, 205), + // Blue blue (0, 0, 255), + BlueViolet blue_violet (138, 43, 226), + Brown brown (165, 42, 42), + BurlyWood burly_wood (222, 184, 135), + CadetBlue cadet_blue (95, 158, 160), + Chartreuse chartreuse (127, 255, 0), + Chocolate chocolate (210, 105, 30), + Coral coral (255, 127, 80), + CornflowerBlue cornflower_blue (100, 149, 237), + Cornsilk cornsilk (255, 248, 220), + Crimson crimson (220, 20, 60), + DarkBlue dark_blue (0, 0, 139), + DarkCyan dark_cyan (0, 139, 139), + DarkGoldenRod dark_golden_rod (184, 134, 11), + DarkGray dark_gray (169, 169, 169), + DarkGrey dark_grey (169, 169, 169), + DarkGreen dark_green (0, 100, 0), + DarkKhaki dark_khaki (189, 183, 107), + DarkMagenta dark_magenta (139, 0, 139), + DarkOliveGreen dark_olive_green (85, 107, 47), + DarkOrange dark_orange (255, 140, 0), + DarkOrchid dark_orchid (153, 50, 204), + DarkRed dark_red (139, 0, 0), + DarkSalmon dark_salmon (233, 150, 122), + DarkSeaGreen dark_sea_green (143, 188, 143), + DarkSlateBlue dark_slate_blue (72, 61, 139), + DarkSlateGray dark_slate_gray (47, 79, 79), + DarkSlateGrey dark_slate_grey (47, 79, 79), + DarkTurquoise dark_turquoise (0, 206, 209), + DarkViolet dark_violet (148, 0, 211), + DeepPink deep_pink (255, 20, 147), + DeepSkyBlue deep_sky_blue (0, 191, 255), + DimGray dim_gray (105, 105, 105), + DimGrey dim_grey (105, 105, 105), + DodgerBlue dodger_blue (30, 144, 255), + FireBrick fire_brick (178, 34, 34), + FloralWhite floral_white (255, 250, 240), + ForestGreen forest_green (34, 139, 34), + Fuchsia fuchsia (255, 0, 255), + Gainsboro gainsboro (220, 220, 220), + GhostWhite ghost_white (248, 248, 255), + Gold gold (255, 215, 0), + GoldenRod golden_rod (218, 165, 32), + Gray gray (128, 128, 128), + Grey grey (128, 128, 128), + // Green green (0, 128, 0), + GreenYellow green_yellow (173, 255, 47), + HoneyDew honey_dew (240, 255, 240), + HotPink hot_pink (255, 105, 180), + IndianRed indian_red (205, 92, 92), + Indigo indigo (75, 0, 130), + Ivory ivory (255, 255, 240), + Khaki khaki (240, 230, 140), + Lavender lavender (230, 230, 250), + LavenderBlush lavender_blush (255, 240, 245), + LawnGreen lawn_green (124, 252, 0), + LemonChiffon lemon_chiffon (255, 250, 205), + LightBlue light_blue (173, 216, 230), + LightCoral light_coral (240, 128, 128), + LightCyan light_cyan (224, 255, 255), + LightGoldenRodYellow light_golden_rod_yellow (250, 250, 210), + LightGray light_gray (211, 211, 211), + LightGrey light_grey (211, 211, 211), + LightGreen light_green (144, 238, 144), + LightPink light_pink (255, 182, 193), + LightSalmon light_salmon (255, 160, 122), + LightSeaGreen light_sea_green (32, 178, 170), + LightSkyBlue light_sky_blue (135, 206, 250), + LightSlateGray light_slate_gray (119, 136, 153), + LightSlateGrey light_slate_grey (119, 136, 153), + LightSteelBlue light_steel_blue (176, 196, 222), + LightYellow light_yellow (255, 255, 224), + Lime lime (0, 255, 0), + LimeGreen lime_green (50, 205, 50), + Linen linen (250, 240, 230), + // Magenta magenta (255, 0, 255), + Maroon maroon (128, 0, 0), + MediumAquaMarine medium_aqua_marine (102, 205, 170), + MediumBlue medium_blue (0, 0, 205), + MediumOrchid medium_orchid (186, 85, 211), + MediumPurple medium_purple (147, 112, 219), + MediumSeaGreen medium_sea_green (60, 179, 113), + MediumSlateBlue medium_slate_blue (123, 104, 238), + MediumSpringGreen medium_spring_green (0, 250, 154), + MediumTurquoise medium_turquoise (72, 209, 204), + MediumVioletRed medium_violet_red (199, 21, 133), + MidnightBlue midnight_blue (25, 25, 112), + MintCream mint_cream (245, 255, 250), + MistyRose misty_rose (255, 228, 225), + Moccasin moccasin (255, 228, 181), + NavajoWhite navajo_white (255, 222, 173), + Navy navy (0, 0, 128), + OldLace old_lace (253, 245, 230), + Olive olive (128, 128, 0), + OliveDrab olive_drab (107, 142, 35), + Orange orange (255, 165, 0), + OrangeRed orange_red (255, 69, 0), + Orchid orchid (218, 112, 214), + PaleGoldenRod pale_golden_rod (238, 232, 170), + PaleGreen pale_green (152, 251, 152), + PaleTurquoise pale_turquoise (175, 238, 238), + PaleVioletRed pale_violet_red (219, 112, 147), + PapayaWhip papaya_whip (255, 239, 213), + PeachPuff peach_puff (255, 218, 185), + Peru peru (205, 133, 63), + Pink pink (255, 192, 203), + Plum plum (221, 160, 221), + PowderBlue powder_blue (176, 224, 230), + Purple purple (128, 0, 128), + RebeccaPurple rebecca_purple (102, 51, 153), + // Red red (255, 0, 0), + RosyBrown rosy_brown (188, 143, 143), + RoyalBlue royal_blue (65, 105, 225), + SaddleBrown saddle_brown (139, 69, 19), + Salmon salmon (250, 128, 114), + SandyBrown sandy_brown (244, 164, 96), + SeaGreen sea_green (46, 139, 87), + SeaShell sea_shell (255, 245, 238), + Sienna sienna (160, 82, 45), + Silver silver (192, 192, 192), + SkyBlue sky_blue (135, 206, 235), + SlateBlue slate_blue (106, 90, 205), + SlateGray slate_gray (112, 128, 144), + SlateGrey slate_grey (112, 128, 144), + Snow snow (255, 250, 250), + SpringGreen spring_green (0, 255, 127), + SteelBlue steel_blue (70, 130, 180), + Tan tan (210, 180, 140), + Teal teal (0, 128, 128), + Thistle thistle (216, 191, 216), + Tomato tomato (255, 99, 71), + Turquoise turquoise (64, 224, 208), + Violet violet (238, 130, 238), + Wheat wheat (245, 222, 179), + // White white (255, 255, 255), + WhiteSmoke white_smoke (245, 245, 245), + // Yellow yellow (255, 255, 0), + YellowGreen yellow_green (154, 205, 50), + + // Styles + Bold bold 1, + Dim dim 2, + Italic italic 3, + Underline underline 4, + Blink blink 5, + BlinkFast blink_fast 6, + Reversed reversed 7, + Hidden hidden 8, + StrikeThrough strike_through 9, + } +} diff --git a/crates/colors/src/support.rs b/crates/colors/src/support.rs new file mode 100644 index 0000000..3c3f87d --- /dev/null +++ b/crates/colors/src/support.rs @@ -0,0 +1,126 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(super) const CSI: &str = "\x1b["; +pub(super) const CSE: &str = "m"; + +macro_rules! elements_inner { + ( + $( + $name:ident $_:ident $number:tt + ),* + $(,)? + ) => { + $( + #[derive(Debug)] + pub struct $name<I: Colorize>(I); + + impl<I: Colorize> Colorize for $name<I> { + fn render_into(self, base: &mut String, use_colors: bool) { + elements_inner! {@parse_number $number} + + if use_colors { + base.write_str(CSI).expect("In-memory write"); + base.write_str(NUMBERS).expect("In-memory write"); + base.write_str(CSE).expect("In-memory write"); + } + self.0.render_into(base, use_colors); + // The canvas is resetting the colours again. + } + } + )* + }; + + (@parse_number $single:literal) => { + const NUMBERS: &str = stringify!($single); + }; + (@parse_number ($red:literal, $green:literal, $blue:literal)) => { + const NUMBERS_U8: [u8; 18] = $crate::custom::rgb_to_ansi($red, $green, $blue, $crate::custom::Plane::Fg); + + const NUMBERS: &str = $crate::custom::bytes_to_str(&NUMBERS_U8); + } +} +pub(super) use elements_inner; + +macro_rules! methods_inner { + ( + Colorize + + $( + $struct_name:ident $fn_name:ident $_:tt + ),* + $(,)? + ) => { + $( + fn $fn_name(self) -> $struct_name<Self> { + $struct_name(self) + } + )* + }; + ( + IntoCanvas + + $( + $struct_name:ident $fn_name:ident $_:tt + ),* + $(,)? + ) => { + $( + fn $fn_name(self) -> $struct_name<Canvas<Self>> { + $struct_name(Canvas(self)) + } + )* + }; +} +pub(super) use methods_inner; + +macro_rules! prepend_input { + ( + $( + $existing_macro_name:path as $new_macro_name:ident $(($macro_rule:tt -> $macro_apply:tt))? + ),* + $(,)? + + <shared_input> + $shared_input:tt + ) => { + $( + prepend_input! { + @generate_macro + $existing_macro_name as $new_macro_name $(($macro_rule -> $macro_apply))? + <shared_input> + $shared_input + } + )* + }; + + ( + @generate_macro + $existing_macro_name:path as $new_macro_name:ident $((($($macro_rule:tt)*) -> {$($macro_apply:tt)*}))? + + <shared_input> + { + $( + $shared_input:tt + )* + } + ) => { + macro_rules! $new_macro_name { + ($($($macro_rule)*)?) => { + $existing_macro_name! { + $($($macro_apply)*)? + $($shared_input)* + } + } + } + pub(super) use $new_macro_name; + } +} +pub(crate) use prepend_input; diff --git a/crates/fmt/Cargo.toml b/crates/fmt/Cargo.toml index 7f82a09..f3cf4ad 100644 --- a/crates/fmt/Cargo.toml +++ b/crates/fmt/Cargo.toml @@ -24,7 +24,7 @@ publish = false path = "src/fmt.rs" [dependencies] -unicode-width = "0.2.0" +unicode-width = "0.2.1" [lints] workspace = true diff --git a/crates/libmpv2/CHANGELOG.md b/crates/libmpv2/CHANGELOG.md index dc6f861..a3d14d7 100644 --- a/crates/libmpv2/CHANGELOG.md +++ b/crates/libmpv2/CHANGELOG.md @@ -16,7 +16,7 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. ## Version 3.0.0 -- \[breaking\] Support libmpv version 2.0 (mpv version 0.35.0). Mpv versions \<= +- [breaking] Support libmpv version 2.0 (mpv version 0.35.0). Mpv versions \<= 0.34.0 will no longer be supported. - Add OpenGL rendering @@ -29,10 +29,10 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. ## Version 2.0.0 - Add method `Mpv::with_initializer` to set options before initialization -- \[breaking\] Borrow `&mut self` in `wait_event` to disallow using two events +- [breaking] Borrow `&mut self` in `wait_event` to disallow using two events where the first points to data freed in the second `wait_event` call -- \[breaking\] `PropertyData<'_>` is no longer `Clone` or `PartialEq`, - `Event<'_>` is no longer `Clone` to avoid cloning/comparing `MpvNode` +- [breaking] `PropertyData<'_>` is no longer `Clone` or `PartialEq`, `Event<'_>` + is no longer `Clone` to avoid cloning/comparing `MpvNode` ## Version 1.1.0 diff --git a/crates/libmpv2/Cargo.toml b/crates/libmpv2/Cargo.toml index fb2f5bf..67fbfec 100644 --- a/crates/libmpv2/Cargo.toml +++ b/crates/libmpv2/Cargo.toml @@ -28,7 +28,7 @@ log.workspace = true [dev-dependencies] crossbeam = "0.8" -sdl2 = "0.37.0" +sdl2 = "0.38.0" [features] default = ["protocols", "render"] diff --git a/crates/libmpv2/examples/opengl.rs b/crates/libmpv2/examples/opengl.rs index 8eb9647..9f595aa 100644 --- a/crates/libmpv2/examples/opengl.rs +++ b/crates/libmpv2/examples/opengl.rs @@ -38,13 +38,16 @@ fn main() { Ok(()) }) .unwrap(); - let mut render_context = RenderContext::new(unsafe { mpv.ctx.as_mut() }, vec![ - RenderParam::ApiType(RenderParamApiType::OpenGl), - RenderParam::InitParams(OpenGLInitParams { - get_proc_address, - ctx: video, - }), - ]) + let mut render_context = RenderContext::new( + unsafe { mpv.ctx.as_mut() }, + vec![ + RenderParam::ApiType(RenderParamApiType::OpenGl), + RenderParam::InitParams(OpenGLInitParams { + get_proc_address, + ctx: video, + }), + ], + ) .expect("Failed creating render context"); event_subsystem diff --git a/crates/libmpv2/libmpv2-sys/Cargo.toml b/crates/libmpv2/libmpv2-sys/Cargo.toml index b0514b8..96141d3 100644 --- a/crates/libmpv2/libmpv2-sys/Cargo.toml +++ b/crates/libmpv2/libmpv2-sys/Cargo.toml @@ -23,4 +23,4 @@ rust-version.workspace = true publish = false [build-dependencies] -bindgen = { version = "0.71.1" } +bindgen = { version = "0.72.0" } diff --git a/crates/libmpv2/libmpv2-sys/build.rs b/crates/libmpv2/libmpv2-sys/build.rs index bf9a02e..45c2450 100644 --- a/crates/libmpv2/libmpv2-sys/build.rs +++ b/crates/libmpv2/libmpv2-sys/build.rs @@ -30,7 +30,9 @@ fn main() { ), "--verbose", ]) - .generate_comments(true) + // NOTE(@bpeetz): The comments are interpreted as doc-tests, + // which obviously fail, as the code is c. <2025-06-16> + .generate_comments(false) .generate() .expect("Unable to generate bindings"); diff --git a/crates/libmpv2/src/lib.rs b/crates/libmpv2/src/lib.rs index d47e620..f6c2103 100644 --- a/crates/libmpv2/src/lib.rs +++ b/crates/libmpv2/src/lib.rs @@ -35,7 +35,7 @@ use std::os::raw as ctype; pub const MPV_CLIENT_API_MAJOR: ctype::c_ulong = 2; pub const MPV_CLIENT_API_MINOR: ctype::c_ulong = 2; pub const MPV_CLIENT_API_VERSION: ctype::c_ulong = - MPV_CLIENT_API_MAJOR << 16 | MPV_CLIENT_API_MINOR; + (MPV_CLIENT_API_MAJOR << 16) | MPV_CLIENT_API_MINOR; mod mpv; #[cfg(test)] diff --git a/crates/libmpv2/src/mpv.rs b/crates/libmpv2/src/mpv.rs index 29dac8d..d8164c0 100644 --- a/crates/libmpv2/src/mpv.rs +++ b/crates/libmpv2/src/mpv.rs @@ -552,21 +552,21 @@ impl Mpv { /// /// # Examples /// - /// ```dont_run - /// # use libmpv2::{Mpv}; - /// # use libmpv2::mpv_node::MpvNode; - /// # use libmpv2::mpv::errors::Result; - /// # use std::collections::HashMap; - /// # - /// # fn main() -> Result<()> { - /// # let mpv = Mpv::new()?; + /// ```text + ///# use libmpv2::{Mpv}; + ///# use libmpv2::mpv_node::MpvNode; + ///# use libmpv2::mpv::errors::Result; + ///# use std::collections::HashMap; + ///# + ///# fn main() -> Result<()> { + ///# let mpv = Mpv::new()?; /// mpv.command("loadfile", &["test-data/jellyfish.mp4", "append-play"]).unwrap(); - /// # let node = mpv.get_property::<MpvNode>("playlist").unwrap(); - /// # let mut list = node.array().unwrap().collect::<Vec<_>>(); - /// # let map = list.pop().unwrap().map().unwrap().collect::<HashMap<_, _>>(); - /// # assert_eq!(map, HashMap::from([(String::from("id"), MpvNode::Int64(1)), (String::from("current"), MpvNode::Flag(true)), (String::from("filename"), MpvNode::String(String::from("test-data/jellyfish.mp4")))])); - /// # Ok(()) - /// # } + ///# let node = mpv.get_property::<MpvNode>("playlist").unwrap(); + ///# let mut list = node.array().unwrap().collect::<Vec<_>>(); + ///# let map = list.pop().unwrap().map().unwrap().collect::<HashMap<_, _>>(); + ///# assert_eq!(map, HashMap::from([(String::from("id"), MpvNode::Int64(1)), (String::from("current"), MpvNode::Flag(true)), (String::from("filename"), MpvNode::String(String::from("test-data/jellyfish.mp4")))])); + ///# Ok(()) + ///# } /// ``` pub fn command(&self, name: &str, args: &[&str]) -> Result<()> { fn escape(input: &str) -> String { diff --git a/crates/libmpv2/src/mpv/events.rs b/crates/libmpv2/src/mpv/events.rs index e27da2c..f10ff6e 100644 --- a/crates/libmpv2/src/mpv/events.rs +++ b/crates/libmpv2/src/mpv/events.rs @@ -70,26 +70,28 @@ impl<'a> PropertyData<'a> { // SAFETY: meant to extract the data from an event property. See `mpv_event_property` in // `client.h` unsafe fn from_raw(format: MpvFormat, ptr: *mut ctype::c_void) -> Result<PropertyData<'a>> { - assert!(!ptr.is_null()); - match format { - mpv_format::Flag => Ok(PropertyData::Flag(*(ptr as *mut bool))), - mpv_format::String => { - let char_ptr = *(ptr as *mut *mut ctype::c_char); - Ok(PropertyData::Str(mpv_cstr_to_str!(char_ptr)?)) - } - mpv_format::OsdString => { - let char_ptr = *(ptr as *mut *mut ctype::c_char); - Ok(PropertyData::OsdStr(mpv_cstr_to_str!(char_ptr)?)) - } - mpv_format::Double => Ok(PropertyData::Double(*(ptr as *mut f64))), - mpv_format::Int64 => Ok(PropertyData::Int64(*(ptr as *mut i64))), - mpv_format::Node => { - let sys_node = *(ptr as *mut libmpv2_sys::mpv_node); - let node = SysMpvNode::new(sys_node, false); - Ok(PropertyData::Node(node.value().unwrap())) + unsafe { + assert!(!ptr.is_null()); + match format { + mpv_format::Flag => Ok(PropertyData::Flag(*(ptr as *mut bool))), + mpv_format::String => { + let char_ptr = *(ptr as *mut *mut ctype::c_char); + Ok(PropertyData::Str(mpv_cstr_to_str!(char_ptr)?)) + } + mpv_format::OsdString => { + let char_ptr = *(ptr as *mut *mut ctype::c_char); + Ok(PropertyData::OsdStr(mpv_cstr_to_str!(char_ptr)?)) + } + mpv_format::Double => Ok(PropertyData::Double(*(ptr as *mut f64))), + mpv_format::Int64 => Ok(PropertyData::Int64(*(ptr as *mut i64))), + mpv_format::Node => { + let sys_node = *(ptr as *mut libmpv2_sys::mpv_node); + let node = SysMpvNode::new(sys_node, false); + Ok(PropertyData::Node(node.value().unwrap())) + } + mpv_format::None => unreachable!(), + _ => unimplemented!(), } - mpv_format::None => unreachable!(), - _ => unimplemented!(), } } } @@ -146,11 +148,13 @@ pub enum Event<'a> { } unsafe extern "C" fn wu_wrapper<F: Fn() + Send + 'static>(ctx: *mut c_void) { - if ctx.is_null() { - panic!("ctx for wakeup wrapper is NULL"); - } + unsafe { + if ctx.is_null() { + panic!("ctx for wakeup wrapper is NULL"); + } - (*(ctx as *mut F))(); + (*(ctx as *mut F))(); + } } /// Context to listen to events. diff --git a/crates/libmpv2/src/mpv/protocol.rs b/crates/libmpv2/src/mpv/protocol.rs index ec840d8..ee33411 100644 --- a/crates/libmpv2/src/mpv/protocol.rs +++ b/crates/libmpv2/src/mpv/protocol.rs @@ -63,26 +63,28 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = user_data as *mut ProtocolData<T, U>; + unsafe { + let data = user_data as *mut ProtocolData<T, U>; - (*info).cookie = user_data; - (*info).read_fn = Some(read_wrapper::<T, U>); - (*info).seek_fn = Some(seek_wrapper::<T, U>); - (*info).size_fn = Some(size_wrapper::<T, U>); - (*info).close_fn = Some(close_wrapper::<T, U>); + (*info).cookie = user_data; + (*info).read_fn = Some(read_wrapper::<T, U>); + (*info).seek_fn = Some(seek_wrapper::<T, U>); + (*info).size_fn = Some(size_wrapper::<T, U>); + (*info).close_fn = Some(close_wrapper::<T, U>); - let ret = panic::catch_unwind(|| { - let uri = mpv_cstr_to_str!(uri as *const _).unwrap(); - ptr::write( - (*data).cookie, - ((*data).open_fn)(&mut (*data).user_data, uri), - ); - }); + let ret = panic::catch_unwind(|| { + let uri = mpv_cstr_to_str!(uri as *const _).unwrap(); + ptr::write( + (*data).cookie, + ((*data).open_fn)(&mut (*data).user_data, uri), + ); + }); - if ret.is_ok() { - 0 - } else { - mpv_error::Generic as _ + if ret.is_ok() { + 0 + } else { + mpv_error::Generic as _ + } } } @@ -95,13 +97,15 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = cookie as *mut ProtocolData<T, U>; + unsafe { + let data = cookie as *mut ProtocolData<T, U>; - let ret = panic::catch_unwind(|| { - let slice = slice::from_raw_parts_mut(buf, nbytes as _); - ((*data).read_fn)(&mut *(*data).cookie, slice) - }); - ret.unwrap_or(-1) + let ret = panic::catch_unwind(|| { + let slice = slice::from_raw_parts_mut(buf, nbytes as _); + ((*data).read_fn)(&mut *(*data).cookie, slice) + }); + ret.unwrap_or(-1) + } } unsafe extern "C" fn seek_wrapper<T, U>(cookie: *mut ctype::c_void, offset: i64) -> i64 @@ -109,18 +113,21 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = cookie as *mut ProtocolData<T, U>; + unsafe { + let data = cookie as *mut ProtocolData<T, U>; - if (*data).seek_fn.is_none() { - return mpv_error::Unsupported as _; - } + if (*data).seek_fn.is_none() { + return mpv_error::Unsupported as _; + } - let ret = - panic::catch_unwind(|| (*(*data).seek_fn.as_ref().unwrap())(&mut *(*data).cookie, offset)); - if let Ok(ret) = ret { - ret - } else { - mpv_error::Generic as _ + let ret = panic::catch_unwind(|| { + (*(*data).seek_fn.as_ref().unwrap())(&mut *(*data).cookie, offset) + }); + if let Ok(ret) = ret { + ret + } else { + mpv_error::Generic as _ + } } } @@ -129,17 +136,20 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = cookie as *mut ProtocolData<T, U>; + unsafe { + let data = cookie as *mut ProtocolData<T, U>; - if (*data).size_fn.is_none() { - return mpv_error::Unsupported as _; - } + if (*data).size_fn.is_none() { + return mpv_error::Unsupported as _; + } - let ret = panic::catch_unwind(|| (*(*data).size_fn.as_ref().unwrap())(&mut *(*data).cookie)); - if let Ok(ret) = ret { - ret - } else { - mpv_error::Unsupported as _ + let ret = + panic::catch_unwind(|| (*(*data).size_fn.as_ref().unwrap())(&mut *(*data).cookie)); + if let Ok(ret) = ret { + ret + } else { + mpv_error::Unsupported as _ + } } } @@ -149,9 +159,11 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = Box::from_raw(cookie as *mut ProtocolData<T, U>); + unsafe { + let data = Box::from_raw(cookie as *mut ProtocolData<T, U>); - panic::catch_unwind(|| (data.close_fn)(Box::from_raw(data.cookie))); + panic::catch_unwind(|| (data.close_fn)(Box::from_raw(data.cookie))); + } } struct ProtocolData<T, U> { @@ -224,20 +236,23 @@ impl<T: RefUnwindSafe, U: RefUnwindSafe> Protocol<T, U> { seek_fn: Option<StreamSeek<T>>, size_fn: Option<StreamSize<T>>, ) -> Protocol<T, U> { - let c_layout = Layout::from_size_align(mem::size_of::<T>(), mem::align_of::<T>()).unwrap(); - let cookie = alloc::alloc(c_layout) as *mut T; - let data = Box::into_raw(Box::new(ProtocolData { - cookie, - user_data, + unsafe { + let c_layout = + Layout::from_size_align(mem::size_of::<T>(), mem::align_of::<T>()).unwrap(); + let cookie = alloc::alloc(c_layout) as *mut T; + let data = Box::into_raw(Box::new(ProtocolData { + cookie, + user_data, - open_fn, - close_fn, - read_fn, - seek_fn, - size_fn, - })); + open_fn, + close_fn, + read_fn, + seek_fn, + size_fn, + })); - Protocol { name, data } + Protocol { name, data } + } } fn register(&self, ctx: *mut libmpv2_sys::mpv_handle) -> Result<()> { diff --git a/crates/libmpv2/src/mpv/render.rs b/crates/libmpv2/src/mpv/render.rs index 6457048..02f70bb 100644 --- a/crates/libmpv2/src/mpv/render.rs +++ b/crates/libmpv2/src/mpv/render.rs @@ -125,26 +125,30 @@ impl<C> From<&RenderParam<C>> for u32 { } unsafe extern "C" fn gpa_wrapper<GLContext>(ctx: *mut c_void, name: *const i8) -> *mut c_void { - if ctx.is_null() { - panic!("ctx for get_proc_address wrapper is NULL"); - } + unsafe { + if ctx.is_null() { + panic!("ctx for get_proc_address wrapper is NULL"); + } - let params: *mut OpenGLInitParams<GLContext> = ctx as _; - let params = &*params; - (params.get_proc_address)( - ¶ms.ctx, - CStr::from_ptr(name) - .to_str() - .expect("Could not convert function name to str"), - ) + let params: *mut OpenGLInitParams<GLContext> = ctx as _; + let params = &*params; + (params.get_proc_address)( + ¶ms.ctx, + CStr::from_ptr(name) + .to_str() + .expect("Could not convert function name to str"), + ) + } } unsafe extern "C" fn ru_wrapper<F: Fn() + Send + 'static>(ctx: *mut c_void) { - if ctx.is_null() { - panic!("ctx for render_update wrapper is NULL"); - } + unsafe { + if ctx.is_null() { + panic!("ctx for render_update wrapper is NULL"); + } - (*(ctx as *mut F))(); + (*(ctx as *mut F))(); + } } impl<C> From<OpenGLInitParams<C>> for libmpv2_sys::mpv_opengl_init_params { @@ -197,14 +201,18 @@ impl<C> From<RenderParam<C>> for libmpv2_sys::mpv_render_param { } unsafe fn free_void_data<T>(ptr: *mut c_void) { - drop(Box::<T>::from_raw(ptr as *mut T)); + unsafe { + drop(Box::<T>::from_raw(ptr as *mut T)); + } } unsafe fn free_init_params<C>(ptr: *mut c_void) { - let params = Box::from_raw(ptr as *mut libmpv2_sys::mpv_opengl_init_params); - drop(Box::from_raw( - params.get_proc_address_ctx as *mut OpenGLInitParams<C>, - )); + unsafe { + let params = Box::from_raw(ptr as *mut libmpv2_sys::mpv_opengl_init_params); + drop(Box::from_raw( + params.get_proc_address_ctx as *mut OpenGLInitParams<C>, + )); + } } impl RenderContext { diff --git a/crates/libmpv2/src/tests.rs b/crates/libmpv2/src/tests.rs index 6106eb2..68753fc 100644 --- a/crates/libmpv2/src/tests.rs +++ b/crates/libmpv2/src/tests.rs @@ -54,10 +54,10 @@ fn properties() { 0.6, f64::round(subg * f64::powi(10.0, 4)) / f64::powi(10.0, 4) ); - mpv.command("loadfile", &[ - "test-data/speech_12kbps_mb.wav", - "append-play", - ]) + mpv.command( + "loadfile", + &["test-data/speech_12kbps_mb.wav", "append-play"], + ) .unwrap(); thread::sleep(Duration::from_millis(250)); @@ -185,10 +185,10 @@ fn events() { fn node_map() { let mpv = Mpv::new().unwrap(); - mpv.command("loadfile", &[ - "test-data/speech_12kbps_mb.wav", - "append-play", - ]) + mpv.command( + "loadfile", + &["test-data/speech_12kbps_mb.wav", "append-play"], + ) .unwrap(); thread::sleep(Duration::from_millis(250)); @@ -217,10 +217,10 @@ fn node_map() { fn node_array() -> Result<()> { let mpv = Mpv::new()?; - mpv.command("loadfile", &[ - "test-data/speech_12kbps_mb.wav", - "append-play", - ]) + mpv.command( + "loadfile", + &["test-data/speech_12kbps_mb.wav", "append-play"], + ) .unwrap(); thread::sleep(Duration::from_millis(250)); diff --git a/crates/libmpv2/update.sh b/crates/libmpv2/update.sh index ecd5aa8..591684a 100755 --- a/crates/libmpv2/update.sh +++ b/crates/libmpv2/update.sh @@ -10,8 +10,4 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -cd "$(dirname "$0")" || exit 1 -[ "$1" = "upgrade" ] && cargo upgrade --incompatible -cargo update - ./libmpv2-sys/update.sh "$@" diff --git a/yt/Cargo.toml b/crates/yt/Cargo.toml index 6f6e470..0b6c581 100644 --- a/yt/Cargo.toml +++ b/crates/yt/Cargo.toml @@ -24,42 +24,44 @@ rust-version.workspace = true publish = false [dependencies] -anyhow = "1.0.96" -blake3 = "1.6.0" -chrono = { version = "0.4.39", features = ["now"] } +anyhow = "1.0.98" +blake3 = { version = "1.8.2", features = ["serde"] } +chrono = { version = "0.4.41", features = ["now"] } chrono-humanize = "0.2.3" -clap = { version = "4.5.30", features = ["derive"] } +clap = { version = "4.5.41", features = ["derive"] } +clap_complete = { version = "4.5.55", features = ["unstable-dynamic"] } +colors.workspace = true futures = "0.3.31" -nucleo-matcher = "0.3.1" -owo-colors = "4.1.0" -regex = "1.11.1" -sqlx = { version = "0.8.3", features = ["runtime-tokio", "sqlite"] } -stderrlog = "0.6.0" -tempfile = "3.17.1" -toml = "0.8.20" -trinitry = { version = "0.2.2" } -xdg = "2.5.2" -bytes.workspace = true libmpv2.workspace = true log.workspace = true +notify = { version = "8.1.0", default-features = false } +regex = "1.11.1" serde.workspace = true serde_json.workspace = true +shlex = "1.3.0" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "sqlite"] } +stderrlog = "0.6.0" +tempfile = "3.20.0" +termsize.workspace = true +tokio-util = { version = "0.7.15", features = ["rt"] } tokio.workspace = true +toml = "0.9.2" url.workspace = true -yt_dlp.workspace = true -termsize.workspace = true uu_fmt.workspace = true -notify = { version = "8.0.0", default-features = false } +xdg = "3.0.0" +yt_dlp.workspace = true +reqwest = "0.12.22" [[bin]] name = "yt" doc = false path = "src/main.rs" -[dev-dependencies] - [lints] workspace = true +[dev-dependencies] +pretty_assertions = "1.4.1" + [package.metadata.docs.rs] all-features = true diff --git a/crates/yt/src/ansi_escape_codes.rs b/crates/yt/src/ansi_escape_codes.rs new file mode 100644 index 0000000..28a8370 --- /dev/null +++ b/crates/yt/src/ansi_escape_codes.rs @@ -0,0 +1,29 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// see: https://en.wikipedia.org/wiki/ANSI_escape_code#Control_Sequence_Introducer_commands +const CSI: &str = "\x1b["; +pub(crate) fn erase_from_cursor_to_bottom() { + print!("{CSI}0J"); +} +pub(crate) fn cursor_up(number: usize) { + // HACK(@bpeetz): The default is `1` and running this command with a + // number of `0` results in it using the default (i.e., `1`) <2025-03-25> + if number != 0 { + print!("{CSI}{number}A"); + } +} + +pub(crate) fn clear_whole_line() { + eprint!("{CSI}2K"); +} +pub(crate) fn move_to_col(x: usize) { + eprint!("{CSI}{x}G"); +} diff --git a/yt/src/app.rs b/crates/yt/src/app.rs index 15a9388..3ea12a4 100644 --- a/yt/src/app.rs +++ b/crates/yt/src/app.rs @@ -16,13 +16,13 @@ use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; use crate::{config::Config, storage::migrate::migrate_db}; #[derive(Debug)] -pub struct App { - pub database: SqlitePool, - pub config: Config, +pub(crate) struct App { + pub(crate) database: SqlitePool, + pub(crate) config: Config, } impl App { - pub async fn new(config: Config, should_migrate_db: bool) -> Result<Self> { + pub(crate) async fn new(config: Config, should_migrate_db: bool) -> Result<Self> { let options = SqliteConnectOptions::new() .filename(&config.paths.database_path) .optimize_on_close(true, None) diff --git a/crates/yt/src/cli.rs b/crates/yt/src/cli.rs new file mode 100644 index 0000000..9a24403 --- /dev/null +++ b/crates/yt/src/cli.rs @@ -0,0 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::PathBuf; + +use clap::{ArgAction, Parser}; + +use crate::commands::Command; + +#[derive(Parser, Debug)] +#[clap(author, about, long_about = None)] +#[allow(clippy::module_name_repetitions)] +/// An command line interface to select, download and watch videos +pub(crate) struct CliArgs { + #[command(subcommand)] + /// The subcommand to execute [default: select] + pub(crate) command: Option<Command>, + + /// Show the version and exit + #[arg(long, short = 'V', action= ArgAction::SetTrue)] + pub(crate) version: bool, + + /// Do not perform database migration before starting. + /// Setting this could cause runtime database access errors. + #[arg(long, short, action=ArgAction::SetTrue, default_value_t = false)] + pub(crate) no_migrate_db: bool, + + /// Display colors [defaults to true, if the config file has no value] + #[arg(long, short = 'C')] + pub(crate) color: Option<bool>, + + /// Set the path to the videos.db. This overrides the default and the config file. + #[arg(long, short)] + pub(crate) db_path: Option<PathBuf>, + + /// Set the path to the config.toml. + /// This overrides the default. + #[arg(long, short)] + pub(crate) config_path: Option<PathBuf>, + + /// Increase message verbosity + #[arg(long="verbose", short = 'v', action = ArgAction::Count)] + pub(crate) verbosity: u8, + + /// Silence all output + #[arg(long, short = 'q')] + pub(crate) quiet: bool, +} + +#[cfg(test)] +mod test { + use clap::CommandFactory; + + use super::CliArgs; + #[test] + fn verify_cli() { + CliArgs::command().debug_assert(); + } +} diff --git a/crates/yt/src/commands/cache/implm.rs b/crates/yt/src/commands/cache/implm.rs new file mode 100644 index 0000000..fd0fbce --- /dev/null +++ b/crates/yt/src/commands/cache/implm.rs @@ -0,0 +1,40 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::CacheCommand, + storage::db::{ + insert::Operations, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +impl CacheCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + CacheCommand::Clear {} => { + let mut videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("Cache clear"); + + for vid in &mut videos { + vid.remove_download_path(&mut ops); + } + + ops.commit(app).await?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/cache/mod.rs b/crates/yt/src/commands/cache/mod.rs new file mode 100644 index 0000000..4ed4b40 --- /dev/null +++ b/crates/yt/src/commands/cache/mod.rs @@ -0,0 +1,19 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Subcommand; + +mod implm; + +#[derive(Debug, Subcommand)] +pub(super) enum CacheCommand { + /// Remove all downloaded video files. + Clear {}, +} diff --git a/crates/yt/src/commands/config/implm.rs b/crates/yt/src/commands/config/implm.rs new file mode 100644 index 0000000..00c28a9 --- /dev/null +++ b/crates/yt/src/commands/config/implm.rs @@ -0,0 +1,23 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{app::App, commands::config::ConfigCommand}; + +use anyhow::Result; + +impl ConfigCommand { + pub(in crate::commands) fn implm(self, app: &App) -> Result<()> { + let config_str = toml::to_string(&app.config)?; + + print!("{config_str}"); + + Ok(()) + } +} diff --git a/yt/src/constants.rs b/crates/yt/src/commands/config/mod.rs index 0f5b918..503b4f7 100644 --- a/yt/src/constants.rs +++ b/crates/yt/src/commands/config/mod.rs @@ -1,6 +1,5 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> // Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // @@ -9,4 +8,9 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -pub const HELP_STR: &str = include_str!("./select/selection_file/help.str"); +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct ConfigCommand {} diff --git a/crates/yt/src/commands/database/implm.rs b/crates/yt/src/commands/database/implm.rs new file mode 100644 index 0000000..07d346b --- /dev/null +++ b/crates/yt/src/commands/database/implm.rs @@ -0,0 +1,45 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::DatabaseCommand, + storage::db::{ + insert::{Committable, subscription, video}, + txn_log::TxnLog, + }, +}; + +use anyhow::Result; + +impl DatabaseCommand { + pub(in crate::commands) async fn implm(&self, app: &App) -> Result<()> { + match self { + DatabaseCommand::Log { kind } => match kind { + super::OperationType::Video => { + let log = TxnLog::<video::Operation>::get(app).await?; + display_log(&log); + } + super::OperationType::Subscription => { + let log = TxnLog::<subscription::Operation>::get(app).await?; + display_log(&log); + } + }, + } + + Ok(()) + } +} + +fn display_log<O: Committable>(log: &TxnLog<O>) { + for (time, value) in log.inner() { + println!("At {time}: {value:?}"); + } +} diff --git a/crates/yt/src/commands/database/mod.rs b/crates/yt/src/commands/database/mod.rs new file mode 100644 index 0000000..06e3169 --- /dev/null +++ b/crates/yt/src/commands/database/mod.rs @@ -0,0 +1,41 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{self, Display}; + +use clap::{Subcommand, ValueEnum}; + +mod implm; + +#[derive(Subcommand, Debug)] +pub(super) enum DatabaseCommand { + /// Show the history of operations, in they groups they were committed in. + Log { + /// What kind of operation to show. + #[arg(short, long, default_value_t)] + kind: OperationType, + }, +} + +#[derive(Debug, Clone, Copy, ValueEnum, Default)] +pub(super) enum OperationType { + #[default] + Video, + Subscription, +} + +impl Display for OperationType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OperationType::Video => f.write_str("video"), + OperationType::Subscription => f.write_str("subscription"), + } + } +} diff --git a/crates/yt/src/commands/download/implm/download/download_options.rs b/crates/yt/src/commands/download/implm/download/download_options.rs new file mode 100644 index 0000000..15fed7e --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/download_options.rs @@ -0,0 +1,121 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Context; +use serde_json::{Value, json}; +use yt_dlp::{YoutubeDL, options::YoutubeDLOptions}; + +use crate::app::App; + +use super::progress_hook::wrapped_progress_hook; + +pub(crate) fn download_opts( + app: &App, + subtitle_langs: Option<&String>, +) -> anyhow::Result<YoutubeDL> { + YoutubeDLOptions::new() + .with_progress_hook(wrapped_progress_hook) + .set("extract_flat", "in_playlist") + .set( + "extractor_args", + json! { + { + "youtube": { + "comment_sort": [ "top" ], + "max_comments": [ "150", "all", "100" ] + } + } + }, + ) + //.set("cookiesfrombrowser", json! {("firefox", "me.google", None::<String>, "youtube_dlp")}) + .set("prefer_free_formats", true) + .set("ffmpeg_location", env!("FFMPEG_LOCATION")) + .set("format", "bestvideo[height<=?1080]+bestaudio/best") + .set("fragment_retries", 10) + .set("getcomments", true) + .set("ignoreerrors", false) + .set("retries", 10) + .set("writeinfojson", true) + // NOTE: This results in a constant warning message. <2025-01-04> + //.set("writeannotations", true) + .set("writesubtitles", true) + .set("writeautomaticsub", true) + .set( + "outtmpl", + json! { + { + "default": app.config.paths.download_dir.join("%(channel)s/%(title)s.%(ext)s"), + "chapter": "%(title)s - %(section_number)03d %(section_title)s [%(id)s].%(ext)s" + } + }, + ) + .set("compat_opts", json! {{}}) + .set("forceprint", json! {{}}) + .set("print_to_file", json! {{}}) + .set("windowsfilenames", false) + .set("restrictfilenames", false) + .set("trim_file_names", false) + .set( + "postprocessors", + json! { + [ + { + "api": "https://sponsor.ajay.app", + "categories": [ + "interaction", + "intro", + "music_offtopic", + "sponsor", + "outro", + "poi_highlight", + "preview", + "selfpromo", + "filler", + "chapter" + ], + "key": "SponsorBlock", + "when": "after_filter" + }, + { + "force_keyframes": false, + "key": "ModifyChapters", + "remove_chapters_patterns": [], + "remove_ranges": [], + "remove_sponsor_segments": [ "sponsor" ], + "sponsorblock_chapter_title": "[SponsorBlock]: %(category_names)l" + }, + { + "add_chapters": true, + "add_infojson": null, + "add_metadata": false, + "key": "FFmpegMetadata" + }, + { + "key": "FFmpegConcat", + "only_multi_video": true, + "when": "playlist" + } + ] + }, + ) + .set( + "subtitleslangs", + Value::Array( + subtitle_langs + .map_or("", String::as_str) + .split(',') + .map(|val| Value::String(val.to_owned())) + .collect::<Vec<_>>(), + ), + ) + .build() + .context("Failed to instanciate download yt_dlp") +} diff --git a/crates/yt/src/commands/download/implm/download/mod.rs b/crates/yt/src/commands/download/implm/download/mod.rs new file mode 100644 index 0000000..876d6e6 --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/mod.rs @@ -0,0 +1,290 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; + +use crate::{ + app::App, + commands::download::implm::download::download_options::download_opts, + shared::bytes::Bytes, + storage::{ + db::{extractor_hash::ExtractorHash, insert::Operations, video::Video}, + notify::{wait_for_cache_reduction, wait_for_db_write}, + }, + yt_dlp::get_current_cache_allocation, +}; + +use anyhow::{Context, Result, bail}; +use log::{debug, error, info, warn}; +use tokio::{select, task::JoinHandle, time}; +use yt_dlp::YoutubeDL; + +#[allow(clippy::module_name_repetitions)] +pub(crate) mod download_options; +pub(crate) mod progress_hook; + +#[derive(Debug)] +#[allow(clippy::module_name_repetitions)] +pub(crate) struct CurrentDownload { + task_handle: JoinHandle<Result<(PathBuf, Video)>>, + yt_dlp: Arc<YoutubeDL>, + extractor_hash: ExtractorHash, +} + +impl CurrentDownload { + fn new_from_video(app: &App, video: Video) -> Result<Self> { + let extractor_hash = video.extractor_hash; + + debug!("Download started: {}", &video.title); + let yt_dlp = Arc::new(download_opts(app, video.subtitle_langs.as_ref())?); + + let local_yt_dlp = Arc::clone(&yt_dlp); + + let task_handle = tokio::task::spawn_blocking(move || { + let mut result = local_yt_dlp + .download(&[video.url.clone()]) + .with_context(|| format!("Failed to download video: '{}'", video.title))?; + + assert_eq!(result.len(), 1); + Ok((result.remove(0), video)) + }); + + Ok(Self { + task_handle, + yt_dlp, + extractor_hash, + }) + } + + fn abort(self) -> Result<()> { + debug!("Cancelling download."); + self.yt_dlp.close()?; + + Ok(()) + } + + fn is_finished(&self) -> bool { + self.task_handle.is_finished() + } + + async fn finalize(self, app: &App) -> Result<()> { + let (result, mut video) = self.task_handle.await??; + + let mut ops = Operations::new("Downloader: Set download path"); + video.set_download_path(&result, &mut ops); + ops.commit(app) + .await + .with_context(|| format!("Failed to committ download of video: '{}'", video.title))?; + + info!( + "Video '{}' was downlaoded to path: {}", + video.title, + result.display() + ); + + Ok(()) + } +} + +enum CacheSizeCheck { + /// The video can be downloaded + Fits, + + /// The video and the current cache size together would exceed the size + TooLarge, + + /// The video would not even fit into the empty cache + ExceedsMaxCacheSize, +} + +#[derive(Debug)] +pub(crate) struct Downloader { + current_download: Option<CurrentDownload>, + video_size_cache: HashMap<ExtractorHash, u64>, + printed_warning: bool, + cached_cache_allocation: Option<Bytes>, +} + +impl Default for Downloader { + fn default() -> Self { + Self::new() + } +} + +impl Downloader { + #[must_use] + pub(crate) fn new() -> Self { + Self { + current_download: None, + video_size_cache: HashMap::new(), + printed_warning: false, + cached_cache_allocation: None, + } + } + + /// Check if enough cache is available. + /// + /// Will wait for the next cache deletion if not. + async fn is_enough_cache_available( + &mut self, + app: &App, + max_cache_size: u64, + next_video: &Video, + ) -> Result<CacheSizeCheck> { + if let Some(cdownload) = &self.current_download { + if cdownload.extractor_hash == next_video.extractor_hash { + // If the video is already being downloaded it will always fit. Otherwise the + // download would not have been started. + return Ok(CacheSizeCheck::Fits); + } + } + let cache_allocation = get_current_cache_allocation(app).await?; + let video_size = self.get_approx_video_size(next_video)?; + + if video_size >= max_cache_size { + error!( + "The video '{}' ({}) exceeds the maximum cache size ({})! \ + Please set a bigger maximum (`--max-cache-size`) or skip it.", + next_video.title, + Bytes::new(video_size), + Bytes::new(max_cache_size) + ); + + return Ok(CacheSizeCheck::ExceedsMaxCacheSize); + } + + if cache_allocation.as_u64() + video_size >= max_cache_size { + if !self.printed_warning { + warn!( + "Can't download video: '{}' ({}) as it's too large for the cache ({} of {} allocated). \ + Waiting for cache size reduction..", + next_video.title, + Bytes::new(video_size), + &cache_allocation, + Bytes::new(max_cache_size) + ); + self.printed_warning = true; + + // Update this value immediately. + // This avoids printing the "Current cache size has changed .." warning below. + self.cached_cache_allocation = Some(cache_allocation); + } + + if let Some(cca) = self.cached_cache_allocation { + if cca != cache_allocation { + // Only print the warning if the display string has actually changed. + // Otherwise, we might confuse the user + if cca.to_string() != cache_allocation.to_string() { + warn!("Current cache size has changed, it's now: '{cache_allocation}'"); + } + debug!( + "Cache size has changed: {} -> {}", + cca.as_u64(), + cache_allocation.as_u64() + ); + self.cached_cache_allocation = Some(cache_allocation); + } + } else { + unreachable!( + "The `printed_warning` should be false in this case, \ + and thus should have already set the `cached_cache_allocation`." + ); + } + + // Wait and hope, that a large video is deleted from the cache. + wait_for_cache_reduction(app).await?; + Ok(CacheSizeCheck::TooLarge) + } else { + self.printed_warning = false; + Ok(CacheSizeCheck::Fits) + } + } + + /// The entry point to the Downloader. + /// This Downloader will periodically check if the database has changed, and then also + /// change which videos it downloads. + /// This will run, until the database doesn't contain any watchable videos + pub(crate) async fn consume(&mut self, app: Arc<App>, max_cache_size: u64) -> Result<()> { + while let Some(next_video) = Video::next_to_download(&app).await? { + match self + .is_enough_cache_available(&app, max_cache_size, &next_video) + .await? + { + CacheSizeCheck::Fits => (), + CacheSizeCheck::TooLarge => continue, + CacheSizeCheck::ExceedsMaxCacheSize => bail!("Giving up."), + } + + if self.current_download.is_some() { + let current_download = self.current_download.take().expect("It is `Some`."); + + if current_download.is_finished() { + // The download is done, finalize it and leave it removed. + current_download.finalize(&app).await?; + continue; + } + + if next_video.extractor_hash == current_download.extractor_hash { + // We still want to download the same video. + // reset the taken value + self.current_download = Some(current_download); + } else { + info!( + "Noticed, that the next video is not the video being downloaded, replacing it ('{}' vs. '{}')!", + next_video.extractor_hash.as_short_hash(&app).await?, + current_download.extractor_hash.as_short_hash(&app).await? + ); + + // Replace the currently downloading video + current_download + .abort() + .context("Failed to abort last download")?; + + let new_current_download = CurrentDownload::new_from_video(&app, next_video)?; + + self.current_download = Some(new_current_download); + } + } else { + info!( + "No video is being downloaded right now, setting it to '{}'", + next_video.title + ); + let new_current_download = CurrentDownload::new_from_video(&app, next_video)?; + self.current_download = Some(new_current_download); + } + + // We have to continuously check, if the current download is done. + // As such we simply wait or recheck on the next write to the db. + select! { + () = time::sleep(Duration::from_secs(1)) => (), + Ok(()) = wait_for_db_write(&app) => (), + } + } + + info!("Finished downloading!"); + Ok(()) + } + + fn get_approx_video_size(&mut self, video: &Video) -> Result<u64> { + if let Some(value) = self.video_size_cache.get(&video.extractor_hash) { + Ok(*value) + } else { + let size = video.get_approx_size()?; + + assert_eq!( + self.video_size_cache.insert(video.extractor_hash, size), + None + ); + + Ok(size) + } + } +} diff --git a/crates/yt/src/commands/download/implm/download/progress_hook.rs b/crates/yt/src/commands/download/implm/download/progress_hook.rs new file mode 100644 index 0000000..19fe122 --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/progress_hook.rs @@ -0,0 +1,175 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + io::{Write, stderr}, + process, + sync::atomic::Ordering, +}; + +use colors::{Colorize, IntoCanvas}; +use log::{Level, log_enabled}; +use yt_dlp::{json_cast, json_get, wrap_progress_hook}; + +use crate::{ + ansi_escape_codes::{clear_whole_line, move_to_col}, + config::SHOULD_DISPLAY_COLOR, + select::duration::MaybeDuration, + shared::bytes::Bytes, +}; + +macro_rules! json_get_default { + ($value:expr, $name:literal, $convert:ident, $default:expr) => { + $value.get($name).map_or($default, |v| { + if v == &serde_json::Value::Null { + $default + } else { + json_cast!(@log_key $name, v, $convert) + } + }) + }; +} + +fn format_bytes(bytes: u64) -> String { + let bytes = Bytes::new(bytes); + bytes.to_string() +} + +fn format_speed(speed: f64) -> String { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let bytes = Bytes::new(speed.floor() as u64); + format!("{bytes}/s") +} + +/// # Panics +/// If expectations fail. +#[allow(clippy::needless_pass_by_value)] +pub(crate) fn progress_hook( + input: serde_json::Map<String, serde_json::Value>, +) -> Result<(), std::io::Error> { + // Only add the handler, if the log-level is higher than Debug (this avoids covering debug + // messages). + if log_enabled!(Level::Debug) { + return Ok(()); + } + + let info_dict = json_get!(input, "info_dict", as_object); + + let get_title = || -> String { + match json_get!(info_dict, "ext", as_str) { + "vtt" => { + format!( + "Subtitles ({})", + json_get_default!(info_dict, "name", as_str, "<No Subtitle Language>") + ) + } + "webm" | "mp4" | "mp3" | "m4a" => { + json_get_default!(info_dict, "title", as_str, "<No title>").to_owned() + } + other => panic!("The extension '{other}' is not yet implemented"), + } + }; + + match json_get!(input, "status", as_str) { + "downloading" => { + let elapsed = json_get_default!(input, "elapsed", as_f64, 0.0); + let eta = json_get_default!(input, "eta", as_f64, 0.0); + let speed = json_get_default!(input, "speed", as_f64, 0.0); + + let downloaded_bytes = json_get!(input, "downloaded_bytes", as_u64); + let (total_bytes, bytes_is_estimate): (u64, &'static str) = { + let total_bytes = json_get_default!(input, "total_bytes", as_u64, 0); + + if total_bytes == 0 { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let maybe_estimate = + json_get_default!(input, "total_bytes_estimate", as_f64, 0.0) as u64; + + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + if maybe_estimate == 0 { + // The download speed should be in bytes + // per second and the eta in seconds. + // Thus multiplying them gets us the raw bytes + // (which were estimated by `yt_dlp`, from their `info.json`) + let bytes_still_needed = (speed * eta).ceil() as u64; + + (downloaded_bytes + bytes_still_needed, "~") + } else { + (maybe_estimate, "~") + } + } else { + (total_bytes, "") + } + }; + + let percent: f64 = { + if total_bytes == 0 { + 100.0 + } else { + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + { + (downloaded_bytes as f64 / total_bytes as f64) * 100.0 + } + } + }; + + clear_whole_line(); + move_to_col(1); + + let should_use_color = SHOULD_DISPLAY_COLOR.load(Ordering::Relaxed); + + eprint!( + "{} [{}/{} at {}] -> [{} of {}{} {}] ", + get_title().bold().blue().render(should_use_color), + MaybeDuration::from_secs_f64(elapsed) + .bold() + .yellow() + .render(should_use_color), + MaybeDuration::from_secs_f64(eta) + .bold() + .yellow() + .render(should_use_color), + format_speed(speed).bold().green().render(should_use_color), + format_bytes(downloaded_bytes) + .bold() + .red() + .render(should_use_color), + bytes_is_estimate.bold().red().render(should_use_color), + format_bytes(total_bytes) + .bold() + .red() + .render(should_use_color), + format!("{percent:.02}%") + .bold() + .cyan() + .render(should_use_color), + ); + stderr().flush()?; + } + "finished" => { + eprintln!("-> Finished downloading."); + } + "error" => { + // TODO: This should probably return an Err. But I'm not so sure where the error would + // bubble up to (i.e., who would catch it) <2025-01-21> + eprintln!("-> Error while downloading: {}", get_title()); + process::exit(1); + } + other => unreachable!("'{other}' should not be a valid state!"), + } + + Ok(()) +} + +wrap_progress_hook!(progress_hook, wrapped_progress_hook); diff --git a/crates/yt/src/commands/download/implm/mod.rs b/crates/yt/src/commands/download/implm/mod.rs new file mode 100644 index 0000000..c74a909 --- /dev/null +++ b/crates/yt/src/commands/download/implm/mod.rs @@ -0,0 +1,55 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::Arc; + +use crate::{ + app::App, + commands::download::DownloadCommand, + shared::bytes::Bytes, + storage::db::{ + insert::{Operations, maintenance::clear_stale_downloaded_paths}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; +use log::info; + +mod download; + +impl DownloadCommand { + pub(in crate::commands) async fn implm(self, app: Arc<App>) -> Result<()> { + let DownloadCommand { + force, + max_cache_size, + } = self; + + let max_cache_size = max_cache_size.unwrap_or(app.config.download.max_cache_size.as_u64()); + info!("Max cache size: '{}'", Bytes::new(max_cache_size)); + + clear_stale_downloaded_paths(&app).await?; + if force { + let mut all = Video::in_states(&app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("Download: Clear old download paths due to `--force`"); + for a in &mut all { + a.remove_download_path(&mut ops); + } + ops.commit(&app).await?; + } + + download::Downloader::new() + .consume(app, max_cache_size) + .await?; + + Ok(()) + } +} diff --git a/crates/yt/src/commands/download/mod.rs b/crates/yt/src/commands/download/mod.rs new file mode 100644 index 0000000..15026ba --- /dev/null +++ b/crates/yt/src/commands/download/mod.rs @@ -0,0 +1,34 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Context; +use clap::Parser; + +use crate::shared::bytes::Bytes; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct DownloadCommand { + /// Forcefully re-download all cached videos (i.e. delete all already downloaded paths, then download). + #[arg(short, long)] + force: bool, + + /// The maximum size the download dir should have. + #[arg(short, long, value_parser = byte_parser)] + max_cache_size: Option<u64>, +} + +fn byte_parser(input: &str) -> Result<u64, anyhow::Error> { + Ok(input + .parse::<Bytes>() + .with_context(|| format!("Failed to parse '{input}' as bytes!"))? + .as_u64()) +} diff --git a/crates/yt/src/commands/implm.rs b/crates/yt/src/commands/implm.rs new file mode 100644 index 0000000..7c60c6a --- /dev/null +++ b/crates/yt/src/commands/implm.rs @@ -0,0 +1,38 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::Arc; + +use crate::commands::Command; + +use anyhow::Result; + +impl Command { + pub(crate) async fn implm(self, app: crate::app::App) -> Result<()> { + match self { + Command::Cache { cmd } => cmd.implm(&app).await?, + Command::Config { cmd } => cmd.implm(&app)?, + Command::Database { cmd } => cmd.implm(&app).await?, + Command::Download { cmd } => cmd.implm(Arc::new(app)).await?, + Command::Playlist { cmd } => cmd.implm(&app).await?, + Command::Select { cmd } => { + cmd.unwrap_or_default().implm(&app).await?; + } + Command::Show { cmd } => cmd.implm(&app).await?, + Command::Status { cmd } => cmd.implm(&app).await?, + Command::Subscriptions { cmd } => cmd.implm(&app).await?, + Command::Update { cmd } => cmd.implm(&app).await?, + Command::Videos { cmd } => cmd.implm(&app).await?, + Command::Watch { cmd } => cmd.implm(Arc::new(app)).await?, + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/mod.rs b/crates/yt/src/commands/mod.rs new file mode 100644 index 0000000..431acef --- /dev/null +++ b/crates/yt/src/commands/mod.rs @@ -0,0 +1,164 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ffi::OsStr, thread}; + +use clap::Subcommand; +use clap_complete::CompletionCandidate; +use tokio::runtime::Runtime; + +use crate::{ + app::App, + commands::{ + cache::CacheCommand, config::ConfigCommand, database::DatabaseCommand, + download::DownloadCommand, playlist::PlaylistCommand, select::SelectCommand, + show::ShowCommand, status::StatusCommand, subscriptions::SubscriptionCommand, + update::UpdateCommand, videos::VideosCommand, watch::WatchCommand, + }, + config::Config, + storage::db::subscription::Subscriptions, +}; + +pub(super) mod implm; + +mod cache; +mod config; +mod database; +mod download; +mod playlist; +mod select; +mod show; +mod status; +mod subscriptions; +mod update; +mod videos; +mod watch; + +#[derive(Subcommand, Debug)] +#[allow(private_interfaces)] // Only the main `implm` method should be accessible. +pub(super) enum Command { + /// Manipulate the download cache + Cache { + #[command(subcommand)] + cmd: CacheCommand, + }, + + /// Show, the configuration options in effect. + Config { + #[command(flatten)] + cmd: ConfigCommand, + }, + + /// Interact with the video database. + #[command(visible_alias = "db")] + Database { + #[command(subcommand)] + cmd: DatabaseCommand, + }, + + /// Download and cache URLs + Download { + #[command(flatten)] + cmd: DownloadCommand, + }, + + /// Visualize the current playlist + Playlist { + #[command(flatten)] + cmd: PlaylistCommand, + }, + + /// Change the state of videos in the database (the default) + Select { + #[command(subcommand)] + cmd: Option<SelectCommand>, + }, + + /// Show things about the currently playing video. + Show { + #[command(subcommand)] + cmd: ShowCommand, + }, + + /// Show, which videos have been selected to be watched (and their cache status) + Status { + #[command(flatten)] + cmd: StatusCommand, + }, + + /// Manipulate subscription + #[command(visible_alias = "subs")] + Subscriptions { + #[command(subcommand)] + cmd: SubscriptionCommand, + }, + + /// Update the video database + Update { + #[command(flatten)] + cmd: UpdateCommand, + }, + + /// Work with single videos + Videos { + #[command(subcommand)] + cmd: VideosCommand, + }, + + /// Watch the already cached (and selected) videos + Watch { + #[command(flatten)] + cmd: WatchCommand, + }, +} + +impl Default for Command { + fn default() -> Self { + Self::Select { + cmd: Some(SelectCommand::default()), + } + } +} + +fn complete_subscription(current: &OsStr) -> Vec<CompletionCandidate> { + let mut output = vec![]; + + let Some(current_prog) = current.to_str().map(ToOwned::to_owned) else { + return output; + }; + + let Ok(config) = Config::from_config_file(None, None, None) else { + return output; + }; + + let handle = thread::spawn(move || { + let Ok(rt) = Runtime::new() else { + return output; + }; + + let Ok(app) = rt.block_on(App::new(config, false)) else { + return output; + }; + + let Ok(all) = rt.block_on(Subscriptions::get(&app)) else { + return output; + }; + + for sub in all.0.into_keys() { + if sub.starts_with(¤t_prog) { + output.push(CompletionCandidate::new(sub)); + } + } + + output + }); + + handle.join().unwrap_or_default() +} diff --git a/crates/yt/src/commands/playlist/implm.rs b/crates/yt/src/commands/playlist/implm.rs new file mode 100644 index 0000000..603184b --- /dev/null +++ b/crates/yt/src/commands/playlist/implm.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{fmt::Write, path::Path}; + +use crate::{ + ansi_escape_codes, + app::App, + commands::playlist::PlaylistCommand, + storage::{ + db::{ + playlist::Playlist, + video::{Video, VideoStatus}, + }, + notify::wait_for_db_write, + }, + videos::RenderWithApp, +}; + +use anyhow::Result; +use futures::{TryStreamExt, stream::FuturesOrdered}; + +impl PlaylistCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let PlaylistCommand { watch } = self; + + let mut previous_output_length = 0; + loop { + let playlist = Playlist::create(app).await?.videos; + + let output = playlist + .into_iter() + .map(|video| async move { + let mut output = String::new(); + + let (_, is_focused) = cache_values(&video); + + if is_focused { + output.push_str("🔻 "); + } else { + output.push_str(" "); + } + + output.push_str(&video.title_fmt().to_string(app)); + + output.push_str(" ("); + output.push_str(&video.parent_subscription_name_fmt().to_string(app)); + output.push(')'); + + output.push_str(" ["); + output.push_str(&video.duration_fmt().to_string(app)); + + if is_focused { + output.push_str(" ("); + if let Some(percent) = video.watch_progress_percent_fmt() { + write!(output, "{}", percent.to_string(app))?; + } else { + write!(output, "{}", video.watch_progress_fmt().to_string(app))?; + } + + output.push(')'); + } + output.push(']'); + + output.push('\n'); + + Ok::<String, anyhow::Error>(output) + }) + .collect::<FuturesOrdered<_>>() + .try_collect::<String>() + .await?; + + // Delete the previous output + ansi_escape_codes::cursor_up(previous_output_length); + ansi_escape_codes::erase_from_cursor_to_bottom(); + + previous_output_length = output.chars().filter(|ch| *ch == '\n').count(); + + print!("{output}"); + + if !watch { + break; + } + + wait_for_db_write(app).await?; + } + + Ok(()) + } +} + +/// Extract the values of the [`VideoStatus::Cached`] value from a Video. +fn cache_values(video: &Video) -> (&Path, bool) { + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + { + (cache_path, *is_focused) + } else { + unreachable!("All of these videos should be cached"); + } +} diff --git a/crates/yt/src/commands/playlist/mod.rs b/crates/yt/src/commands/playlist/mod.rs new file mode 100644 index 0000000..8d3407d --- /dev/null +++ b/crates/yt/src/commands/playlist/mod.rs @@ -0,0 +1,20 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct PlaylistCommand { + /// Linger and display changes + #[arg(short, long)] + watch: bool, +} diff --git a/yt/src/select/selection_file/help.str b/crates/yt/src/commands/select/implm/fs_generators/help.str index e3cc347..e3cc347 100644 --- a/yt/src/select/selection_file/help.str +++ b/crates/yt/src/commands/select/implm/fs_generators/help.str diff --git a/yt/src/select/selection_file/help.str.license b/crates/yt/src/commands/select/implm/fs_generators/help.str.license index a0e196c..a0e196c 100644 --- a/yt/src/select/selection_file/help.str.license +++ b/crates/yt/src/commands/select/implm/fs_generators/help.str.license diff --git a/crates/yt/src/commands/select/implm/fs_generators/mod.rs b/crates/yt/src/commands/select/implm/fs_generators/mod.rs new file mode 100644 index 0000000..10da032 --- /dev/null +++ b/crates/yt/src/commands/select/implm/fs_generators/mod.rs @@ -0,0 +1,355 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::HashMap, + env, + fs::{self, File, OpenOptions}, + io::{BufRead, BufReader, BufWriter, Read, Write}, + iter, + os::fd::{AsFd, AsRawFd}, + path::Path, +}; + +use crate::{ + app::App, + cli::CliArgs, + commands::{ + Command, + select::{ + SelectCommand, SelectSplitSortKey, SelectSplitSortMode, + implm::standalone::{self, handle_select_cmd}, + }, + }, + storage::db::{ + extractor_hash::ExtractorHash, + insert::Operations, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use futures::{TryStreamExt, stream::FuturesOrdered}; +use log::info; +use shlex::Shlex; +use tokio::process; + +const HELP_STR: &str = include_str!("./help.str"); + +pub(crate) async fn select_split( + app: &App, + done: bool, + sort_key: SelectSplitSortKey, + sort_mode: SelectSplitSortMode, +) -> Result<()> { + let temp_dir = tempfile::Builder::new() + .prefix("yt_video_select-") + .rand_bytes(6) + .tempdir() + .context("Failed to get tempdir")?; + + let matching_videos = get_videos(app, done).await?; + + let mut no_author = vec![]; + let mut author_map = HashMap::new(); + for video in matching_videos { + if let Some(sub) = &video.parent_subscription_name { + if author_map.contains_key(sub) { + let vec: &mut Vec<_> = author_map + .get_mut(sub) + .expect("This key is set, we checked in the if above"); + + vec.push(video); + } else { + author_map.insert(sub.to_owned(), vec![video]); + } + } else { + no_author.push(video); + } + } + + let author_map = { + let mut temp_vec: Vec<_> = author_map.into_iter().collect(); + + match sort_key { + SelectSplitSortKey::Publisher => { + // PERFORMANCE: The clone here should not be neeed. <2025-06-15> + temp_vec.sort_by_key(|(name, _): &(String, Vec<Video>)| name.to_owned()); + } + SelectSplitSortKey::Videos => { + temp_vec.sort_by_key(|(_, videos): &(String, Vec<Video>)| videos.len()); + } + } + + match sort_mode { + SelectSplitSortMode::Asc => { + // Std's default mode is ascending. + } + SelectSplitSortMode::Desc => { + temp_vec.reverse(); + } + } + + temp_vec + }; + + for (index, (name, videos)) in author_map + .into_iter() + .chain(iter::once(( + "<No parent subscription>".to_owned(), + no_author, + ))) + .enumerate() + { + let mut file_path = temp_dir.path().join(format!("{index:02}_{name}")); + file_path.set_extension("yts"); + + let tmp_file = File::create(&file_path) + .with_context(|| format!("Falied to create file at: {}", file_path.display()))?; + + write_videos_to_file(app, &tmp_file, &videos) + .await + .with_context(|| format!("Falied to populate file at: {}", file_path.display()))?; + } + + open_editor_at(temp_dir.path()).await?; + + let mut paths = vec![]; + for maybe_entry in temp_dir + .path() + .read_dir() + .context("Failed to open temp dir for reading")? + { + let entry = maybe_entry.context("Failed to read entry in temp dir")?; + + if !entry.file_type()?.is_file() { + bail!("Found non-file entry: {}", entry.path().display()); + } + + paths.push(entry.path()); + } + + paths.sort(); + + let mut persistent_file = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(&app.config.paths.last_selection_path) + .context("Failed to open persistent selection file")?; + + for path in paths { + let mut read_file = File::open(path)?; + + let mut buffer = vec![]; + read_file.read_to_end(&mut buffer)?; + persistent_file.write_all(&buffer)?; + } + + persistent_file.flush()?; + let persistent_file = OpenOptions::new() + .read(true) + .open(format!( + "/proc/self/fd/{}", + persistent_file.as_fd().as_raw_fd() + )) + .context("Failed to re-open persistent file")?; + + let processed = process_file(app, &persistent_file).await?; + + info!("Processed {processed} records."); + temp_dir.close().context("Failed to close the temp dir")?; + Ok(()) +} + +pub(crate) async fn select_file(app: &App, done: bool, use_last_selection: bool) -> Result<()> { + let temp_file = tempfile::Builder::new() + .prefix("yt_video_select-") + .suffix(".yts") + .rand_bytes(6) + .tempfile() + .context("Failed to get tempfile")?; + + if use_last_selection { + fs::copy(&app.config.paths.last_selection_path, &temp_file)?; + } else { + let matching_videos = get_videos(app, done).await?; + + write_videos_to_file(app, temp_file.as_file(), &matching_videos).await?; + } + + open_editor_at(temp_file.path()).await?; + + let read_file = OpenOptions::new().read(true).open(temp_file.path())?; + fs::copy(temp_file.path(), &app.config.paths.last_selection_path) + .context("Failed to persist selection file")?; + + let processed = process_file(app, &read_file).await?; + info!("Processed {processed} records."); + + Ok(()) +} + +async fn get_videos(app: &App, include_done: bool) -> Result<Vec<Video>> { + if include_done { + Video::in_states(app, VideoStatusMarker::ALL).await + } else { + Video::in_states( + app, + &[ + VideoStatusMarker::Pick, + // + VideoStatusMarker::Watch, + VideoStatusMarker::Cached, + ], + ) + .await + } +} + +async fn write_videos_to_file(app: &App, file: &File, videos: &[Video]) -> Result<()> { + // Warm-up the cache for the display rendering of the videos. + // Otherwise the futures would all try to warm it up at the same time. + if let Some(vid) = videos.first() { + drop(vid.to_line_display(app, None).await?); + } + + let mut edit_file = BufWriter::new(file); + + videos + .iter() + .map(|vid| vid.to_select_file_display(app)) + .collect::<FuturesOrdered<_>>() + .try_collect::<Vec<String>>() + .await? + .into_iter() + .try_for_each(|line| -> Result<()> { + edit_file + .write_all(line.as_bytes()) + .context("Failed to write to `edit_file`")?; + + Ok(()) + })?; + + edit_file.write_all(HELP_STR.as_bytes())?; + edit_file.flush().context("Failed to flush edit file")?; + + Ok(()) +} + +async fn process_file(app: &App, file: &File) -> Result<i64> { + let mut line_number = 0; + + let mut ops = Operations::new("Select: process file"); + + // Fetch all the hashes once, instead of every time we need to process a line. + let all_hashes = ExtractorHash::get_all(app).await?; + + let reader = BufReader::new(file); + for line in reader.lines() { + let line = line.context("Failed to read a line")?; + + if let Some(line) = process_line(&line)? { + line_number -= 1; + + // debug!( + // "Parsed command: `{}`", + // line.iter() + // .map(|val| format!("\"{}\"", val)) + // .collect::<Vec<String>>() + // .join(" ") + // ); + + let arg_line = ["yt", "select"] + .into_iter() + .chain(line.iter().map(String::as_str)); + + let args = CliArgs::parse_from(arg_line); + + let Command::Select { cmd } = args + .command + .expect("This will be some, as we constructed it above.") + else { + unreachable!("This is checked in the `filter_line` function") + }; + + match cmd.expect( + "This value should always be some \ + here, as it would otherwise thrown an error above.", + ) { + SelectCommand::File { .. } | SelectCommand::Split { .. } => { + bail!("You cannot use `select file` or `select split` recursively.") + } + SelectCommand::Add { urls, start, stop } => { + Box::pin(standalone::add::add(app, urls, start, stop)).await?; + } + other => { + let shared = other + .clone() + .into_shared() + .expect("The ones without shared should have been filtered out."); + + let hash = shared.hash.realize(app, Some(&all_hashes)).await?; + let mut video = hash + .get_with_app(app) + .await + .expect("The hash was already realized, it should therefore exist"); + + handle_select_cmd(app, other, &mut video, Some(line_number), &mut ops).await?; + } + } + } + } + + ops.commit(app).await?; + Ok(-line_number) +} + +async fn open_editor_at(path: &Path) -> Result<()> { + let editor = env::var("EDITOR").unwrap_or("nvim".to_owned()); + + let mut nvim = process::Command::new(&editor); + nvim.arg(path); + let status = nvim + .status() + .await + .with_context(|| format!("Falied to run editor: {editor}"))?; + + if status.success() { + Ok(()) + } else { + bail!("Editor ({editor}) exited with error status: {}", status) + } +} + +fn process_line(line: &str) -> Result<Option<Vec<String>>> { + // Filter out comments and empty lines + if line.starts_with('#') || line.trim().is_empty() { + Ok(None) + } else { + let split: Vec<_> = { + let mut shl = Shlex::new(line); + let res = shl.by_ref().collect(); + + if shl.had_error { + bail!("Failed to parse line '{line}'") + } + + assert_eq!(shl.line_no, 1, "A unexpected newline appeared"); + res + }; + + assert!(!split.is_empty()); + + Ok(Some(split)) + } +} diff --git a/crates/yt/src/commands/select/implm/mod.rs b/crates/yt/src/commands/select/implm/mod.rs new file mode 100644 index 0000000..f39c77f --- /dev/null +++ b/crates/yt/src/commands/select/implm/mod.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{app::App, commands::select::SelectCommand, storage::db::insert::Operations}; + +use anyhow::Result; + +mod fs_generators; +mod standalone; + +impl SelectCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + SelectCommand::File { + done, + use_last_selection, + } => Box::pin(fs_generators::select_file(app, done, use_last_selection)).await?, + SelectCommand::Split { + done, + sort_key, + sort_mode, + } => Box::pin(fs_generators::select_split(app, done, sort_key, sort_mode)).await?, + SelectCommand::Add { urls, start, stop } => { + Box::pin(standalone::add::add(app, urls, start, stop)).await?; + } + other => { + let shared = other + .clone() + .into_shared() + .expect("The ones without shared should have been filtered out."); + let hash = shared.hash.realize(app, None).await?; + let mut video = hash + .get_with_app(app) + .await + .expect("The hash was already realized, it should therefore exist"); + + let mut ops = Operations::new("Main: handle select cmd"); + standalone::handle_select_cmd(app, other, &mut video, None, &mut ops).await?; + ops.commit(app).await?; + } + } + + Ok(()) + } +} diff --git a/yt/src/select/cmds/add.rs b/crates/yt/src/commands/select/implm/standalone/add.rs index da58ec2..dd11cb4 100644 --- a/yt/src/select/cmds/add.rs +++ b/crates/yt/src/commands/select/implm/standalone/add.rs @@ -10,39 +10,28 @@ use crate::{ app::App, - download::download_options::download_opts, - storage::video_database::{ - self, extractor_hash::ExtractorHash, get::get_all_hashes, set::add_video, - }, - unreachable::Unreachable, - update::video_entry_to_video, + storage::db::{extractor_hash::ExtractorHash, insert::Operations, video::Video}, + yt_dlp::yt_dlp_opts_updating, }; use anyhow::{Context, Result, bail}; use log::{error, warn}; -use serde_json::{Map, Value}; use url::Url; -use yt_dlp::wrapper::info_json::InfoType; +use yt_dlp::{YoutubeDL, info_json::InfoJson, json_cast, json_get, json_try_get}; #[allow(clippy::too_many_lines)] -pub(super) async fn add( +pub(crate) async fn add( app: &App, urls: Vec<Url>, start: Option<usize>, stop: Option<usize>, ) -> Result<()> { for url in urls { - async fn process_and_add( - app: &App, - entry: yt_dlp::wrapper::info_json::InfoJson, - opts: &Map<String, Value>, - ) -> Result<()> { - let url = entry - .url - .unreachable("`yt_dlp` should guarantee that this is Some at this point"); - - let entry = yt_dlp::extract_info(opts, &url, false, true) - .await + async fn process_and_add(app: &App, entry: InfoJson, yt_dlp: &YoutubeDL) -> Result<()> { + let url = json_get!(entry, "url", as_str).parse()?; + + let entry = yt_dlp + .extract_info(&url, false, true) .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; add_entry(app, entry).await?; @@ -50,56 +39,47 @@ pub(super) async fn add( Ok(()) } - async fn add_entry(app: &App, entry: yt_dlp::wrapper::info_json::InfoJson) -> Result<()> { + async fn add_entry(app: &App, entry: InfoJson) -> Result<()> { // We have to re-fetch all hashes every time, because a user could try to add the same // URL twice (for whatever reason.) - let hashes = get_all_hashes(app) + let hashes = ExtractorHash::get_all(app) .await .context("Failed to fetch all video hashes")?; - let extractor_hash = blake3::hash( - entry - .id - .as_ref() - .expect("This should be some at this point") - .as_bytes(), - ); + + let extractor_hash = ExtractorHash::from_info_json(&entry); if hashes.contains(&extractor_hash) { error!( "Video '{}'{} is already in the database. Skipped adding it", - ExtractorHash::from_hash(extractor_hash) - .into_short_hash(app) + extractor_hash + .as_short_hash(app) .await .with_context(|| format!( "Failed to format hash of video '{}' as short hash", - entry - .url - .map_or("<Unknown video Url>".to_owned(), |url| url.to_string()) + json_try_get!(entry, "url", as_str).unwrap_or("<Unknown video Url>") ))?, - entry - .title - .map_or(String::new(), |title| format!(" ('{title}')")) + json_try_get!(entry, "title", as_str) + .map_or(String::new(), |title| format!(" (\"{title}\")")) ); return Ok(()); } - let video = video_entry_to_video(entry, None)?; - add_video(app, video.clone()).await?; + let mut ops = Operations::new("SelectAdd: Video entry to video"); + let video = Video::from_info_json(&entry, None)?.add(&mut ops)?; + ops.commit(app).await?; - println!("{}", &video.to_line_display(app).await?); + println!("{}", &video.to_line_display(app, None).await?); Ok(()) } - let opts = download_opts(app, &video_database::YtDlpOptions { - subtitle_langs: String::new(), - }); + let yt_dlp = yt_dlp_opts_updating(start.unwrap_or(0) + stop.unwrap_or(0))?; - let entry = yt_dlp::extract_info(&opts, &url, false, true) - .await + let entry = yt_dlp + .extract_info(&url, false, true) .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; - match entry._type { - Some(InfoType::Video) => { + match json_try_get!(entry, "_type", as_str) { + Some("video") => { add_entry(app, entry).await?; if start.is_some() || stop.is_some() { warn!( @@ -107,13 +87,13 @@ pub(super) async fn add( ); } } - Some(InfoType::Playlist) => { - if let Some(entries) = entry.entries { + Some("playlist") => { + if let Some(entries) = json_try_get!(entry, "entries", as_array) { let start = start.unwrap_or(0); let stop = stop.unwrap_or(entries.len() - 1); - let mut respected_entries: Vec<_> = take_vector(entries, start, stop) - .with_context(|| { + let respected_entries = + take_vector(entries, start, stop).with_context(|| { format!( "Failed to take entries starting at: {start} and ending with {stop}" ) @@ -123,11 +103,23 @@ pub(super) async fn add( warn!("No entries found, after applying your start/stop limits."); } else { // Pre-warm the cache - process_and_add(app, respected_entries.remove(0), &opts).await?; + process_and_add( + app, + json_cast!(respected_entries[0], as_object).to_owned(), + &yt_dlp, + ) + .await?; + let respected_entries = &respected_entries[1..]; let futures: Vec<_> = respected_entries - .into_iter() - .map(|entry| process_and_add(app, entry, &opts)) + .iter() + .map(|entry| { + process_and_add( + app, + json_cast!(entry, as_object).to_owned(), + &yt_dlp, + ) + }) .collect(); for fut in futures { @@ -148,7 +140,7 @@ pub(super) async fn add( Ok(()) } -fn take_vector<T>(vector: Vec<T>, start: usize, stop: usize) -> Result<Vec<T>> { +fn take_vector<T>(vector: &[T], start: usize, stop: usize) -> Result<&[T]> { let length = vector.len(); if stop >= length { @@ -157,37 +149,18 @@ fn take_vector<T>(vector: Vec<T>, start: usize, stop: usize) -> Result<Vec<T>> { ); } - let end_skip = { - let base = length - .checked_sub(stop) - .unreachable("The check above should have caught this case."); - - base.checked_sub(1) - .unreachable("The check above should have caught this case.") - }; - - // NOTE: We're using this instead of the `vector[start..=stop]` notation, because I wanted to - // avoid the needed allocation to turn the slice into a vector. <2025-01-04> - - // TODO: This function could also just return a slice, but oh well.. <2025-01-04> - Ok(vector - .into_iter() - .skip(start) - .rev() - .skip(end_skip) - .rev() - .collect()) + Ok(&vector[start..=stop]) } #[cfg(test)] mod test { - use crate::select::cmds::add::take_vector; + use super::take_vector; #[test] fn test_vector_take() { let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - let new_vec = take_vector(vec, 2, 8).unwrap(); + let new_vec = take_vector(&vec, 2, 8).unwrap(); assert_eq!(new_vec, vec![2, 3, 4, 5, 6, 7, 8]); } @@ -196,13 +169,13 @@ mod test { fn test_vector_take_overflow() { let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert!(take_vector(vec, 0, 12).is_err()); + assert!(take_vector(&vec, 0, 12).is_err()); } #[test] fn test_vector_take_equal() { let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - assert!(take_vector(vec, 0, 11).is_err()); + assert!(take_vector(&vec, 0, 11).is_err()); } } diff --git a/crates/yt/src/commands/select/implm/standalone/mod.rs b/crates/yt/src/commands/select/implm/standalone/mod.rs new file mode 100644 index 0000000..9512e32 --- /dev/null +++ b/crates/yt/src/commands/select/implm/standalone/mod.rs @@ -0,0 +1,132 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::io::{Write, stderr}; + +use crate::{ + ansi_escape_codes, + app::App, + commands::select::{SelectCommand, SharedSelectionCommandArgs}, + storage::db::{ + insert::{Operations, video::Operation}, + video::{Priority, Video, VideoStatus}, + }, +}; + +use anyhow::{Context, Result, bail}; + +pub(super) mod add; + +pub(crate) async fn handle_select_cmd( + app: &App, + cmd: SelectCommand, + video: &mut Video, + line_number: Option<i64>, + ops: &mut Operations<Operation>, +) -> Result<()> { + let status = match cmd { + SelectCommand::Pick { shared } => Some((VideoStatus::Pick, shared)), + SelectCommand::Drop { shared } => Some((VideoStatus::Drop, shared)), + SelectCommand::Watched { shared } => Some((VideoStatus::Watched, shared)), + SelectCommand::Watch { shared } => { + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + { + Some(( + VideoStatus::Cached { + cache_path: cache_path.to_owned(), + is_focused: *is_focused, + }, + shared, + )) + } else { + Some((VideoStatus::Watch, shared)) + } + } + SelectCommand::Url { shared } => { + let Some(url) = shared.url else { + bail!("You need to provide a url to `select url ..`") + }; + + let mut firefox = std::process::Command::new(app.config.commands.url_opener.first()); + firefox.args(app.config.commands.url_opener.tail()); + firefox.arg(url.as_str()); + let _handle = firefox.spawn().context("Failed to run firefox")?; + None + } + SelectCommand::File { .. } | SelectCommand::Split { .. } | SelectCommand::Add { .. } => { + unreachable!("These should have been filtered out") + } + }; + + if let Some((status, shared)) = status { + handle_status_change( + app, + video, + shared, + line_number, + status, + line_number.is_none(), + ops, + ) + .await?; + } + + Ok(()) +} + +async fn handle_status_change( + app: &App, + video: &mut Video, + shared: SharedSelectionCommandArgs, + line_number: Option<i64>, + new_status: VideoStatus, + is_single: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + let priority = compute_priority(line_number, shared.priority); + + video.set_status(new_status, ops); + if let Some(priority) = priority { + video.set_priority(priority, ops); + } + + if let Some(subtitle_langs) = shared.subtitle_langs { + video.set_subtitle_langs(subtitle_langs, ops); + } + if let Some(playback_speed) = shared.playback_speed { + video.set_playback_speed(playback_speed, ops); + } + + if !is_single { + ansi_escape_codes::clear_whole_line(); + ansi_escape_codes::move_to_col(1); + } + + eprint!("{}", &video.to_line_display(app, None).await?); + + if is_single { + eprintln!(); + } else { + stderr().flush()?; + } + + Ok(()) +} + +fn compute_priority(line_number: Option<i64>, priority: Option<i64>) -> Option<Priority> { + if let Some(pri) = priority { + Some(Priority::from(pri)) + } else { + line_number.map(Priority::from) + } +} diff --git a/crates/yt/src/commands/select/mod.rs b/crates/yt/src/commands/select/mod.rs new file mode 100644 index 0000000..db69238 --- /dev/null +++ b/crates/yt/src/commands/select/mod.rs @@ -0,0 +1,230 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fmt::{self, Display, Formatter}, + str::FromStr, +}; + +use chrono::NaiveDate; +use clap::{Args, Subcommand, ValueEnum}; +use url::Url; + +use crate::{select::duration::MaybeDuration, storage::db::extractor_hash::LazyExtractorHash}; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +// NOTE: Keep this in sync with the [`constants::HELP_STR`] constant. <2024-08-20> +// NOTE: Also keep this in sync with the `tree-sitter-yts/grammar.js`. <2024-11-04> +#[allow(private_interfaces)] // Only the main `implm` method should be accessible. +pub(super) enum SelectCommand { + /// Open a `git rebase` like file to select the videos to watch (the default) + File { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Use the last selection file (useful if you've spend time on it and want to get it again) + #[arg(long, short, conflicts_with = "done")] + use_last_selection: bool, + }, + + /// Generate a directory, where each file contains only one subscription. + Split { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Which key to use for sorting. + #[arg(default_value_t)] + sort_key: SelectSplitSortKey, + + /// Which mode to use for sorting. + #[arg(default_value_t)] + sort_mode: SelectSplitSortMode, + }, + + /// Add a video to the database + /// + /// This optionally supports to add a playlist. + /// When a playlist is added, the `start` and `stop` arguments can be used to select which + /// playlist entries to include. + #[command(visible_alias = "a")] + Add { + urls: Vec<Url>, + + /// Start adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 's', long)] + start: Option<usize>, + + /// Stop adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 'e', long)] + stop: Option<usize>, + }, + + /// Mark the video given by the hash to be watched + #[command(visible_alias = "w")] + Watch { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash to be dropped + #[command(visible_alias = "d")] + Drop { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash as already watched + #[command(visible_alias = "wd")] + Watched { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Open the video URL in your specified command + #[command(visible_alias = "u")] + Url { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Reset the videos status to 'Pick' + #[command(visible_alias = "p")] + Pick { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, +} +impl Default for SelectCommand { + fn default() -> Self { + Self::File { + done: false, + use_last_selection: false, + } + } +} + +#[derive(Clone, Debug, Args)] +#[command(infer_subcommands = true)] +/// Mark the video given by the hash to be watched +struct SharedSelectionCommandArgs { + /// The ordering priority (higher means more at the top) + #[arg(short, long)] + priority: Option<i64>, + + /// The subtitles to download (e.g. 'en,de,sv') + #[arg(short = 'l', long)] + subtitle_langs: Option<String>, + + /// The speed to set mpv to + #[arg(short = 's', long)] + playback_speed: Option<f64>, + + /// The short extractor hash + hash: LazyExtractorHash, + + title: Option<String>, + + date: Option<OptionalNaiveDate>, + + publisher: Option<OptionalPublisher>, + + duration: Option<MaybeDuration>, + + url: Option<Url>, +} + +impl SelectCommand { + fn into_shared(self) -> Option<SharedSelectionCommandArgs> { + match self { + SelectCommand::File { .. } + | SelectCommand::Split { .. } + | SelectCommand::Add { .. } => None, + SelectCommand::Watch { shared } + | SelectCommand::Drop { shared } + | SelectCommand::Watched { shared } + | SelectCommand::Url { shared } + | SelectCommand::Pick { shared } => Some(shared), + } + } +} + +#[derive(Clone, Debug, Copy)] +struct OptionalNaiveDate { + date: Option<NaiveDate>, +} +impl FromStr for OptionalNaiveDate { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No release date]" { + Ok(Self { date: None }) + } else { + Ok(Self { + date: Some(NaiveDate::from_str(v)?), + }) + } + } +} +#[derive(Clone, Debug)] +struct OptionalPublisher { + publisher: Option<String>, +} +impl FromStr for OptionalPublisher { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No author]" { + Ok(Self { publisher: None }) + } else { + Ok(Self { + publisher: Some(v.to_owned()), + }) + } + } +} + +#[derive(Default, ValueEnum, Clone, Copy, Debug)] +enum SelectSplitSortKey { + /// Sort by the name of the publisher. + #[default] + Publisher, + + /// Sort by the number of unselected videos per publisher. + Videos, +} +impl Display for SelectSplitSortKey { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + SelectSplitSortKey::Publisher => f.write_str("publisher"), + SelectSplitSortKey::Videos => f.write_str("videos"), + } + } +} + +#[derive(Default, ValueEnum, Clone, Copy, Debug)] +enum SelectSplitSortMode { + /// Sort in ascending order (small -> big) + #[default] + Asc, + + /// Sort in descending order (big -> small) + Desc, +} + +impl Display for SelectSplitSortMode { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + SelectSplitSortMode::Asc => f.write_str("asc"), + SelectSplitSortMode::Desc => f.write_str("desc"), + } + } +} diff --git a/crates/yt/src/commands/show/implm/mod.rs b/crates/yt/src/commands/show/implm/mod.rs new file mode 100644 index 0000000..a2e40fd --- /dev/null +++ b/crates/yt/src/commands/show/implm/mod.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fs::{self, OpenOptions}, + io, + process::Command, +}; + +use crate::{ + app::App, + commands::ShowCommand, + output::{display_fmt_and_less, display_less}, + storage::db::video::Video, +}; + +use anyhow::{Context, Result, anyhow, bail}; +use tempfile::Builder; +use tokio_util::bytes::Buf; + +impl ShowCommand { + pub(in crate::commands) async fn implm(&self, app: &App) -> Result<()> { + match self { + ShowCommand::Description {} => { + let description = Video::get_current_description(app).await?; + + display_fmt_and_less(&description)?; + } + ShowCommand::Comments {} => { + let comments = Video::get_current_comments(app).await?; + + display_less(comments.render(app.config.global.display_colors))?; + } + ShowCommand::Thumbnail {} => { + let video = Video::currently_focused(app).await?.ok_or(anyhow!( + "You need to have a current video to display its info" + ))?; + + if let Some(url) = video.thumbnail_url { + let response = reqwest::get(url.clone()) + .await + .with_context(|| format!("Failed to download thumbnail from url: {url}"))?; + let response = response + .error_for_status() + .context("Failed to download thumbnail")?; + + let (tmp_path, mut tmp) = { + let file = Builder::new().prefix("yt-thumbnail-download").tempfile()?; + let (_, path) = file.keep()?; + let new_file = OpenOptions::new() + .write(true) + .read(false) + .create(false) + .truncate(true) + .open(&path)?; + + (path, new_file) + }; + + let mut content = response.bytes().await?.reader(); + io::copy(&mut content, &mut tmp)?; + + let status = Command::new(app.config.commands.image_show.first()) + .args(app.config.commands.image_show.tail()) + .arg(tmp_path.as_os_str()) + .status() + .context("Failed to spawn image show command")?; + + if !status.success() { + bail!( + "{:?} failed with status: {}", + &app.config.commands.image_show.join(" "), + status + ); + } + + fs::remove_file(&tmp_path).with_context(|| { + format!( + "Failed to cleanup downloaded thumbnail image at: {}", + tmp_path.display() + ) + })?; + } else { + eprintln!("Current video does not have a thumbnail."); + } + } + ShowCommand::Info {} => { + let video = Video::currently_focused(app).await?.ok_or(anyhow!( + "You need to have a current video to display its info" + ))?; + + display_less( + video + .to_info_display(app, None) + .await + .context("Failed to format video")?, + )?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/show/mod.rs b/crates/yt/src/commands/show/mod.rs new file mode 100644 index 0000000..60f2e51 --- /dev/null +++ b/crates/yt/src/commands/show/mod.rs @@ -0,0 +1,30 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Subcommand; + +mod implm; + +#[derive(Subcommand, Debug)] +pub(super) enum ShowCommand { + /// Display the description of the currently playing video + Description {}, + + /// Display the comments of the currently playing video. + Comments {}, + + /// Display the thumbnail of the currently playing video. + Thumbnail {}, + + /// Display general info of the currently playing video. + /// + /// This is the same as running `yt videos info <hash of current video>` + Info {}, +} diff --git a/crates/yt/src/commands/status/implm.rs b/crates/yt/src/commands/status/implm.rs new file mode 100644 index 0000000..dabc5df --- /dev/null +++ b/crates/yt/src/commands/status/implm.rs @@ -0,0 +1,157 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::time::Duration; + +use crate::{ + app::App, + commands::status::StatusCommand, + select::duration::MaybeDuration, + shared::bytes::Bytes, + storage::db::{ + subscription::Subscriptions, + video::{Video, VideoStatusMarker}, + }, + yt_dlp::get_current_cache_allocation, +}; + +use anyhow::{Context, Result}; + +macro_rules! get { + ($videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .count() + }; + + (@collect $videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .collect() + }; +} + +impl StatusCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let StatusCommand { format } = self; + + let all_videos = Video::in_states(app, VideoStatusMarker::ALL).await?; + + // lengths + let picked_videos_len = get!(all_videos, Pick); + + let watch_videos_len = get!(all_videos, Watch); + let cached_videos_len = get!(all_videos, Cached); + let watched_videos_len = get!(all_videos, Watched); + let watched_videos: Vec<_> = get!(@collect all_videos, Watched); + + let drop_videos_len = get!(all_videos, Drop); + let dropped_videos_len = get!(all_videos, Dropped); + + let subscriptions = Subscriptions::get(app).await?; + let subscriptions_len = subscriptions.0.len(); + + let watchtime_status = { + let total_watch_time_raw = watched_videos + .iter() + .fold(Duration::default(), |acc, vid| acc + vid.watch_progress); + + // Most things are watched at a speed of s (which is defined in the config file). + // Thus + // y = x * s -> y / s = x + let total_watch_time = Duration::from_secs_f64( + (total_watch_time_raw.as_secs_f64()) / app.config.select.playback_speed, + ); + + let speed = app.config.select.playback_speed; + + // Do not print the adjusted time, if the user has keep the speed level at 1. + #[allow(clippy::float_cmp)] + if speed == 1.0 { + format!( + "Total Watchtime: {}\n", + MaybeDuration::from_std(total_watch_time_raw) + ) + } else { + format!( + "Total Watchtime: {} (at {speed} speed: {})\n", + MaybeDuration::from_std(total_watch_time_raw), + MaybeDuration::from_std(total_watch_time), + ) + } + }; + + let watch_rate: f64 = { + fn to_f64(input: usize) -> f64 { + f64::from(u32::try_from(input).expect("This should never exceed u32::MAX")) + } + + let count = + to_f64(watched_videos_len) / (to_f64(drop_videos_len) + to_f64(dropped_videos_len)); + count * 100.0 + }; + + let cache_usage: Bytes = get_current_cache_allocation(app) + .await + .context("Failed to get current cache allocation")?; + + if let Some(fmt) = format { + let output = fmt + .replace( + "{picked_videos_len}", + picked_videos_len.to_string().as_str(), + ) + .replace("{watch_videos_len}", watch_videos_len.to_string().as_str()) + .replace( + "{cached_videos_len}", + cached_videos_len.to_string().as_str(), + ) + .replace( + "{watched_videos_len}", + watched_videos_len.to_string().as_str(), + ) + .replace("{watch_rate}", watch_rate.to_string().as_str()) + .replace("{drop_videos_len}", drop_videos_len.to_string().as_str()) + .replace( + "{dropped_videos_len}", + dropped_videos_len.to_string().as_str(), + ) + .replace("{watchtime_status}", watchtime_status.to_string().as_str()) + .replace( + "{subscriptions_len}", + subscriptions_len.to_string().as_str(), + ) + .replace("{cache_usage}", cache_usage.to_string().as_str()); + + print!("{output}"); + } else { + println!( + "\ +Picked Videos: {picked_videos_len} + +Watch Videos: {watch_videos_len} +Cached Videos: {cached_videos_len} +Watched Videos: {watched_videos_len} (watch rate: {watch_rate:.2} %) + +Drop Videos: {drop_videos_len} +Dropped Videos: {dropped_videos_len} + +{watchtime_status} + + Subscriptions: {subscriptions_len} + Cache usage: {cache_usage}" + ); + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/status/mod.rs b/crates/yt/src/commands/status/mod.rs new file mode 100644 index 0000000..4a8dee7 --- /dev/null +++ b/crates/yt/src/commands/status/mod.rs @@ -0,0 +1,20 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct StatusCommand { + /// Which format to use + #[arg(short, long)] + format: Option<String>, +} diff --git a/crates/yt/src/commands/subscriptions/implm.rs b/crates/yt/src/commands/subscriptions/implm.rs new file mode 100644 index 0000000..31b714e --- /dev/null +++ b/crates/yt/src/commands/subscriptions/implm.rs @@ -0,0 +1,253 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::str::FromStr; + +use crate::{ + app::App, + commands::subscriptions::SubscriptionCommand, + storage::db::{ + insert::{Operations, subscription::Operation}, + subscription::{Subscription, Subscriptions, check_url}, + }, +}; + +use anyhow::{Context, Result, bail}; +use log::{error, warn}; +use tokio::{ + fs::File, + io::{AsyncBufRead, AsyncBufReadExt, BufReader, stdin}, +}; +use url::Url; +use yt_dlp::{json_cast, json_get, json_try_get, options::YoutubeDLOptions}; + +impl SubscriptionCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + SubscriptionCommand::Add { + name, + url, + no_check, + } => { + let mut ops = Operations::new("main: subscribe"); + subscribe(app, name, url, no_check, &mut ops) + .await + .context("Failed to add a subscription")?; + ops.commit(app).await?; + } + SubscriptionCommand::Remove { name } => { + let mut present_subscriptions = Subscriptions::get(app).await?; + + let mut ops = Operations::new("Subscribe: unsubscribe"); + if let Some(subscription) = present_subscriptions.0.remove(&name) { + subscription.remove(&mut ops); + } else { + bail!("Couldn't find subscription: '{}'", &name); + } + ops.commit(app) + .await + .with_context(|| format!("Failed to unsubscribe from {name:?}"))?; + } + SubscriptionCommand::List {} => { + let all_subs = Subscriptions::get(app).await?; + + for (key, val) in all_subs.0 { + println!("{}: '{}'", key, val.url); + } + } + SubscriptionCommand::Export {} => { + let all_subs = Subscriptions::get(app).await?; + for val in all_subs.0.values() { + println!("{}", val.url); + } + } + SubscriptionCommand::Import { + file, + force, + no_check, + } => { + if let Some(file) = file { + let f = File::open(file).await?; + + import(app, BufReader::new(f), force, no_check).await?; + } else { + import(app, BufReader::new(stdin()), force, no_check).await?; + } + } + } + + Ok(()) + } +} + +async fn import<W: AsyncBufRead + AsyncBufReadExt + Unpin>( + app: &App, + reader: W, + force: bool, + no_check: bool, +) -> Result<()> { + let mut ops = Operations::new("SubscribeImport: init"); + + let all = Subscriptions::get(app).await?; + if force { + all.remove(&mut ops); + } + ops.commit(app).await?; + let mut ops = Operations::new("SubscribeImport: after all subs remove"); + + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + let url = + Url::from_str(&line).with_context(|| format!("Failed to parse '{line}' as url"))?; + + match subscribe(app, None, url, no_check, &mut ops) + .await + .with_context(|| format!("Failed to subscribe to: '{line}'")) + { + Ok(()) => (), + Err(err) => eprintln!( + "Error while subscribing to '{}': '{}'", + line, + err.source().expect("Should have a source") + ), + } + } + ops.commit(app).await?; + + Ok(()) +} + +async fn subscribe( + app: &App, + name: Option<String>, + url: Url, + no_check: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + if !(url.as_str().ends_with("videos") + || url.as_str().ends_with("streams") + || url.as_str().ends_with("shorts") + || url.as_str().ends_with("videos/") + || url.as_str().ends_with("streams/") + || url.as_str().ends_with("shorts/")) + && url.as_str().contains("youtube.com") + { + warn!( + "Your youtube url does not seem like it actually tracks a channels playlist \ + (videos, streams, shorts). Adding subscriptions for each of them..." + ); + + let url = Url::parse(&(url.as_str().to_owned() + "/")) + .expect("This was an url, it should stay one"); + + let (videos, streams, shorts) = if let Some(name) = name { + ( + Some(name.clone() + " {Videos}"), + Some(name.clone() + " {Streams}"), + Some(name.clone() + " {Shorts}"), + ) + } else { + (None, None, None) + }; + + let _ = actual_subscribe( + app, + videos, + url.join("videos/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!("Failed to subscribe to the '{}' variant: {err}", "{Videos}"); + }); + + let _ = actual_subscribe( + app, + streams, + url.join("streams/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!( + "Failed to subscribe to the '{}' variant: {err}", + "{Streams}" + ); + }); + + let _ = actual_subscribe( + app, + shorts, + url.join("shorts/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!("Failed to subscribe to the '{}' variant: {err}", "{Shorts}"); + }); + } else { + actual_subscribe(app, name, url, no_check, ops).await?; + } + + Ok(()) +} + +async fn actual_subscribe( + app: &App, + name: Option<String>, + url: Url, + no_check: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + if !no_check && !check_url(url.clone()).await? { + bail!("The url ('{}') does not represent a playlist!", &url) + } + + let name = if let Some(name) = name { + name + } else { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + if json_try_get!(info, "_type", as_str) == Some("playlist") { + json_get!(info, "title", as_str).to_owned() + } else { + bail!("The url ('{}') does not represent a playlist!", &url) + } + }; + + let present_subscriptions = Subscriptions::get(app).await?; + + if let Some(subs) = present_subscriptions.0.get(&name) { + bail!( + "The subscription '{}' could not be added, \ + as another one with the same name ('{}') already exists. \ + It points to the Url: '{}'", + name, + name, + subs.url + ); + } + + let sub = Subscription { name, url }; + + sub.add(ops); + + Ok(()) +} diff --git a/crates/yt/src/commands/subscriptions/mod.rs b/crates/yt/src/commands/subscriptions/mod.rs new file mode 100644 index 0000000..edd41c6 --- /dev/null +++ b/crates/yt/src/commands/subscriptions/mod.rs @@ -0,0 +1,62 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::PathBuf; + +use clap::Subcommand; +use clap_complete::ArgValueCompleter; +use url::Url; + +use crate::commands::complete_subscription; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +pub(super) enum SubscriptionCommand { + /// Subscribe to an URL + Add { + #[arg(short, long)] + /// The human readable name of the subscription + name: Option<String>, + + /// The URL to listen to + url: Url, + + /// Don't check, whether the URL actually points to something yt understands. + #[arg(long, default_value_t = false)] + no_check: bool, + }, + + /// Unsubscribe from an URL + Remove { + /// The human readable name of the subscription + #[arg(add = ArgValueCompleter::new(complete_subscription))] + name: String, + }, + + /// Import a bunch of URLs as subscriptions. + Import { + /// The file containing the URLs. Will use Stdin otherwise. + file: Option<PathBuf>, + + /// Remove any previous subscriptions + #[arg(short, long)] + force: bool, + + /// Don't check, whether the URLs actually point to something yt understands. + #[arg(long, default_value_t = false)] + no_check: bool, + }, + /// Write all subscriptions in an format understood by `import` + Export {}, + + /// List all subscriptions + List {}, +} diff --git a/crates/yt/src/commands/update/implm/mod.rs b/crates/yt/src/commands/update/implm/mod.rs new file mode 100644 index 0000000..53c7415 --- /dev/null +++ b/crates/yt/src/commands/update/implm/mod.rs @@ -0,0 +1,62 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::update::{UpdateCommand, implm::updater::Updater}, + storage::db::{ + extractor_hash::ExtractorHash, + subscription::{Subscription, Subscriptions}, + }, +}; + +use anyhow::{Result, bail}; + +mod updater; + +impl UpdateCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let UpdateCommand { + max_backlog, + subscriptions: subscription_names_to_update, + } = self; + + let mut all_subs = Subscriptions::get(app).await?; + + let max_backlog = max_backlog.unwrap_or(app.config.update.max_backlog); + + let subs: Vec<Subscription> = if subscription_names_to_update.is_empty() { + all_subs.0.into_values().collect() + } else { + subscription_names_to_update + .into_iter() + .map(|sub| { + if let Some(val) = all_subs.0.remove(&sub) { + Ok(val) + } else { + bail!( + "Your specified subscription to update '{}' is not a subscription!", + sub + ) + } + }) + .collect::<Result<_>>()? + }; + + // We can get away with not having to re-fetch the hashes every time, as the returned video + // should not contain duplicates. + let hashes = ExtractorHash::get_all(app).await?; + + let updater = Updater::new(max_backlog, app.config.update.pool_size, hashes); + updater.update(app, subs).await?; + + Ok(()) + } +} diff --git a/crates/yt/src/commands/update/implm/updater.rs b/crates/yt/src/commands/update/implm/updater.rs new file mode 100644 index 0000000..2b96bf2 --- /dev/null +++ b/crates/yt/src/commands/update/implm/updater.rs @@ -0,0 +1,205 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use anyhow::{Context, Result}; +use futures::{StreamExt, future::join_all, stream}; +use log::{Level, debug, error, log_enabled}; +use tokio::io::{AsyncWriteExt, stderr}; +use tokio_util::task::LocalPoolHandle; +use yt_dlp::{ + info_json::InfoJson, json_cast, json_try_get, options::YoutubeDLOptions, process_ie_result, + python_error::PythonError, +}; + +use crate::{ + ansi_escape_codes, + app::App, + storage::db::{ + extractor_hash::ExtractorHash, insert::Operations, subscription::Subscription, video::Video, + }, + yt_dlp::yt_dlp_opts_updating, +}; + +pub(super) struct Updater { + max_backlog: usize, + hashes: Vec<ExtractorHash>, + pool: LocalPoolHandle, +} + +static REACHED_NUMBER: AtomicUsize = const { AtomicUsize::new(1) }; + +impl Updater { + pub(super) fn new(max_backlog: usize, max_threads: usize, hashes: Vec<ExtractorHash>) -> Self { + let pool = LocalPoolHandle::new(max_threads); + + Self { + max_backlog, + hashes, + pool, + } + } + + pub(super) async fn update(self, app: &App, subscriptions: Vec<Subscription>) -> Result<()> { + let total_number = subscriptions.len(); + + let mut stream = stream::iter(subscriptions) + .map(|sub| self.get_new_entries(sub, total_number)) + .buffer_unordered(app.config.update.futures); + + while let Some(output) = stream.next().await { + let mut entries = output?; + + if let Some(next) = entries.next() { + let (sub, entry) = next; + process_subscription(app, sub, entry).await?; + + join_all(entries.map(|(sub, entry)| process_subscription(app, sub, entry))) + .await + .into_iter() + .collect::<Result<(), _>>()?; + } + } + + Ok(()) + } + + #[allow(clippy::too_many_lines)] + async fn get_new_entries( + &self, + sub: Subscription, + total_number: usize, + ) -> Result<impl Iterator<Item = (Subscription, InfoJson)>> { + let max_backlog = self.max_backlog; + let hashes = self.hashes.clone(); + + let yt_dlp = yt_dlp_opts_updating(max_backlog)?; + + self.pool + .spawn_pinned(move || { + async move { + if !log_enabled!(Level::Debug) { + ansi_escape_codes::clear_whole_line(); + ansi_escape_codes::move_to_col(1); + eprint!( + "({}/{total_number}) Checking playlist {}...", + REACHED_NUMBER.fetch_add(1, Ordering::Relaxed), + sub.name + ); + ansi_escape_codes::move_to_col(1); + stderr().flush().await?; + } + + let info = yt_dlp + .extract_info(&sub.url, false, false) + .with_context(|| format!("Failed to get playlist '{}'.", sub.name))?; + + let empty = vec![]; + let entries = json_try_get!(info, "entries", as_array).unwrap_or(&empty); + + let valid_entries: Vec<(Subscription, InfoJson)> = entries + .iter() + .take(max_backlog) + .filter_map(|entry| -> Option<(Subscription, InfoJson)> { + let extractor_hash = + ExtractorHash::from_info_json(json_cast!(entry, as_object)); + + if hashes.contains(&extractor_hash) { + debug!( + "Skipping entry, as it is \ + already present: '{extractor_hash}'", + ); + None + } else { + Some((sub.clone(), json_cast!(entry, as_object).to_owned())) + } + }) + .collect(); + + Ok(valid_entries + .into_iter() + .map(|(sub, entry)| { + let inner_yt_dlp = YoutubeDLOptions::new() + .set("noplaylist", true) + .build() + .expect("Worked before, should work now"); + + match inner_yt_dlp.process_ie_result(entry, false) { + Ok(output) => Ok((sub, output)), + Err(err) => Err(err), + } + }) + // Don't fail the whole update, if one of the entries fails to fetch. + .filter_map(move |base| match base { + Ok(ok) => Some(ok), + Err(err) => { + match err { + process_ie_result::Error::Python(PythonError(err)) => { + if err.contains( + "Join this channel to get access \ + to members-only content ", + ) { + // Hide this error + } else { + // Show the error, but don't fail. + let error = err + .strip_prefix( + "DownloadError: \u{1b}[0;31mERROR:\u{1b}[0m ", + ) + .unwrap_or(&err); + error!("While fetching {:#?}: {error}", sub.name); + } + + None + } + process_ie_result::Error::InfoJsonPrepare(error) => { + error!( + "While fetching {:#?}: Failed to prepare \ + info json: {error}", + sub.name + ); + None + } + } + } + })) + } + }) + .await? + } +} + +async fn process_subscription(app: &App, sub: Subscription, entry: InfoJson) -> Result<()> { + let mut ops = Operations::new("Update: process subscription"); + let video = Video::from_info_json(&entry, Some(&sub)) + .context("Failed to parse search entry as Video")?; + + let title = video.title.clone(); + let url = video.url.clone(); + let video = video.add(&mut ops).with_context(|| { + format!("Failed to add video to database: '{title}' (with url: '{url}')") + })?; + + ops.commit(app).await.with_context(|| { + format!( + "Failed to add video to database: '{}' (with url: '{}')", + video.title, video.url + ) + })?; + println!( + "{}", + &video + .to_line_display(app, None) + .await + .with_context(|| format!("Failed to format video: '{}'", video.title))? + ); + Ok(()) +} diff --git a/crates/yt/src/commands/update/mod.rs b/crates/yt/src/commands/update/mod.rs new file mode 100644 index 0000000..cb29148 --- /dev/null +++ b/crates/yt/src/commands/update/mod.rs @@ -0,0 +1,27 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; +use clap_complete::ArgValueCompleter; + +use crate::commands::complete_subscription; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct UpdateCommand { + /// The maximal number of videos to fetch for each subscription. + #[arg(short, long)] + max_backlog: Option<usize>, + + /// The subscriptions to update + #[arg(add = ArgValueCompleter::new(complete_subscription))] + subscriptions: Vec<String>, +} diff --git a/crates/yt/src/commands/videos/implm.rs b/crates/yt/src/commands/videos/implm.rs new file mode 100644 index 0000000..2a018c7 --- /dev/null +++ b/crates/yt/src/commands/videos/implm.rs @@ -0,0 +1,73 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::videos::VideosCommand, + storage::db::video::{Video, VideoStatusMarker}, +}; + +use anyhow::{Context, Result}; +use futures::{TryStreamExt, stream::FuturesUnordered}; + +impl VideosCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + VideosCommand::List { + search_query, + limit, + format, + } => { + let all_videos = Video::in_states(app, VideoStatusMarker::ALL).await?; + + // turn one video to a color display, to pre-warm the hash shrinking cache + if let Some(val) = all_videos.first() { + val.to_line_display(app, format.clone()).await?; + } + + let limit = limit.unwrap_or(all_videos.len()); + + let all_video_strings: Vec<String> = all_videos + .into_iter() + .take(limit) + .map(|vid| to_line_display_owned(vid, app, format.clone())) + .collect::<FuturesUnordered<_>>() + .try_collect::<Vec<String>>() + .await?; + + if let Some(query) = search_query { + all_video_strings + .into_iter() + .filter(|video| video.to_lowercase().contains(&query.to_lowercase())) + .for_each(|video| println!("{video}")); + } else { + println!("{}", all_video_strings.join("\n")); + } + } + VideosCommand::Info { hash, format } => { + let video = hash.realize(app, None).await?.get_with_app(app).await?; + + print!( + "{}", + &video + .to_info_display(app, format) + .await + .context("Failed to format video")? + ); + } + } + + Ok(()) + } +} + +async fn to_line_display_owned(video: Video, app: &App, format: Option<String>) -> Result<String> { + video.to_line_display(app, format).await +} diff --git a/crates/yt/src/commands/videos/mod.rs b/crates/yt/src/commands/videos/mod.rs new file mode 100644 index 0000000..ca20715 --- /dev/null +++ b/crates/yt/src/commands/videos/mod.rs @@ -0,0 +1,46 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::{ArgAction, Subcommand}; + +use crate::storage::db::extractor_hash::LazyExtractorHash; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +pub(super) enum VideosCommand { + /// List the videos in the database + #[command(visible_alias = "ls")] + List { + /// An optional search query to limit the results + #[arg(action = ArgAction::Append)] + search_query: Option<String>, + + /// The format string to use. + // TODO(@bpeetz): Encode the default format, as the default string here. <2025-07-04> + #[arg(short, long)] + format: Option<String>, + + /// The number of videos to show + #[arg(short, long)] + limit: Option<usize>, + }, + + /// Get detailed information about a video + Info { + /// The short hash of the video + hash: LazyExtractorHash, + + /// The format string to use. + // TODO(@bpeetz): Encode the default format, as the default string here. <2025-07-04> + #[arg(short, long)] + format: Option<String>, + }, +} diff --git a/crates/yt/src/commands/watch/implm/mod.rs b/crates/yt/src/commands/watch/implm/mod.rs new file mode 100644 index 0000000..8182216 --- /dev/null +++ b/crates/yt/src/commands/watch/implm/mod.rs @@ -0,0 +1,244 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fs, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use crate::{ + app::App, + commands::watch::{WatchCommand, implm::playlist_handler::Status}, + storage::{ + db::{ + insert::{Operations, maintenance::clear_stale_downloaded_paths}, + playlist::Playlist, + }, + notify::wait_for_db_write, + }, +}; + +use anyhow::{Context, Result}; +use libmpv2::{Mpv, events::EventContext}; +use log::{debug, info, trace, warn}; +use tokio::{task, time}; + +mod playlist_handler; + +impl WatchCommand { + #[allow(clippy::too_many_lines)] + pub(in crate::commands) async fn implm(self, app: Arc<App>) -> Result<()> { + let WatchCommand { + provide_ipc_socket, + headless, + } = self; + + clear_stale_downloaded_paths(&app).await?; + + let ipc_socket = if provide_ipc_socket { + Some(app.config.paths.mpv_ipc_socket_path.clone()) + } else { + None + }; + + let (mpv, mut ev_ctx) = + init_mpv(&app, ipc_socket, headless).context("Failed to initialize mpv instance")?; + let mpv = Arc::new(mpv); + + if provide_ipc_socket { + println!("{}", app.config.paths.mpv_ipc_socket_path.display()); + } + + let should_break = Arc::new(AtomicBool::new(false)); + let local_app = Arc::clone(&app); + let local_mpv = Arc::clone(&mpv); + let local_should_break = Arc::clone(&should_break); + let progress_handle = task::spawn(async move { + loop { + if local_should_break.load(Ordering::Relaxed) { + trace!("WatchProgressThread: Stopping, as we received exit signal."); + break; + } + + let mut playlist = Playlist::create(&local_app).await?; + + if let Some(index) = playlist.current_index() { + trace!("WatchProgressThread: Saving watch progress for current video"); + + let mut ops = + Operations::new("WatchProgressThread: save watch progress thread"); + playlist.save_watch_progress(&local_mpv, index, &mut ops); + ops.commit(&local_app).await?; + } else { + trace!( + "WatchProgressThread: Tried to save current watch progress, but no video active." + ); + } + + time::sleep(local_app.config.watch.progress_save_intervall).await; + } + + Ok::<(), anyhow::Error>(()) + }); + + let playlist = Playlist::create(&app).await?; + playlist.resync_with_mpv(&app, &mpv)?; + + let mut have_warned = (false, 0); + 'watchloop: loop { + 'waitloop: while let Ok(value) = playlist_handler::status(&app).await { + match value { + Status::NoMoreAvailable => { + break 'watchloop; + } + Status::NoCached { marked_watch } => { + // try again next time. + if have_warned.0 { + if have_warned.1 != marked_watch { + warn!("Now {marked_watch} videos are marked as to be watched."); + have_warned.1 = marked_watch; + } + } else { + warn!( + "There is nothing to watch yet, but still {marked_watch} videos marked as to be watched. \ + Will idle, until they become available" + ); + have_warned = (true, marked_watch); + } + wait_for_db_write(&app).await?; + + // Add the new videos, if they are there. + let playlist = Playlist::create(&app).await?; + playlist.resync_with_mpv(&app, &mpv)?; + } + Status::Available { newly_available } => { + debug!( + "Checked for currently available videos and found {newly_available}!" + ); + have_warned.0 = false; + + // Something just became available! + break 'waitloop; + } + } + } + + // TODO(@bpeetz): Is the following assumption correct? <2025-07-10> + // We wait until forever for the next event, because we really don't need to do anything + // else. + if let Some(ev) = ev_ctx.wait_event(f64::MAX) { + match ev { + Ok(event) => { + trace!("Mpv event triggered: {event:#?}"); + if playlist_handler::handle_mpv_event(&app, &mpv, &event) + .await + .with_context(|| format!("Failed to handle mpv event: '{event:#?}'"))? + { + break; + } + } + Err(e) => debug!("Mpv Event errored: {e}"), + } + } + } + should_break.store(true, Ordering::Relaxed); + progress_handle.await??; + + if provide_ipc_socket { + fs::remove_file(&app.config.paths.mpv_ipc_socket_path).with_context(|| { + format!( + "Failed to clean-up the mpv ipc socket at {}", + app.config.paths.mpv_ipc_socket_path.display() + ) + })?; + } + + Ok(()) + } +} + +fn init_mpv(app: &App, ipc_socket: Option<PathBuf>, headless: bool) -> Result<(Mpv, EventContext)> { + // set some default values, to make things easier (these can be overridden by the config file, + // which we load later) + let mpv = Mpv::with_initializer(|mpv| { + if let Some(socket) = ipc_socket { + mpv.set_property( + "input-ipc-server", + socket + .to_str() + .expect("This path comes from us, it should never contain not-utf8"), + )?; + } + + if headless { + // Do not provide video output. + mpv.set_property("vid", "no")?; + } else { + // Enable default key bindings, so the user can actually interact with + // the player (and e.g. close the window). + mpv.set_property("input-default-bindings", "yes")?; + mpv.set_property("input-vo-keyboard", "yes")?; + + // Show the on screen controller. + mpv.set_property("osc", "yes")?; + + // Don't automatically advance to the next video (or exit the player) + mpv.set_option("keep-open", "always")?; + + // Always display an window, even for non-video playback. + // As mpv does not have cli access, no window means no control and no user feedback. + mpv.set_option("force-window", "yes")?; + } + + Ok(()) + }) + .context("Failed to initialize mpv")?; + + let config_path = &app.config.paths.mpv_config_path; + if config_path.try_exists()? { + info!("Found mpv.conf at '{}'!", config_path.display()); + mpv.command( + "load-config-file", + &[config_path + .to_str() + .context("Failed to parse the config path is utf8-stringt")?], + )?; + } else { + warn!( + "Did not find a mpv.conf file at '{}'", + config_path.display() + ); + } + + let input_path = &app.config.paths.mpv_input_path; + if input_path.try_exists()? { + info!("Found mpv.input.conf at '{}'!", input_path.display()); + mpv.command( + "load-input-conf", + &[input_path + .to_str() + .context("Failed to parse the input path as utf8 string")?], + )?; + } else { + warn!( + "Did not find a mpv.input.conf file at '{}'", + input_path.display() + ); + } + + let ev_ctx = EventContext::new(mpv.ctx); + ev_ctx.disable_deprecated_events()?; + + Ok((mpv, ev_ctx)) +} diff --git a/yt/src/watch/playlist_handler/client_messages/mod.rs b/crates/yt/src/commands/watch/implm/playlist_handler/client_messages.rs index 6f7a59e..fd7e035 100644 --- a/yt/src/watch/playlist_handler/client_messages/mod.rs +++ b/crates/yt/src/commands/watch/implm/playlist_handler/client_messages.rs @@ -10,7 +10,7 @@ use std::{env, time::Duration}; -use crate::{app::App, comments}; +use crate::{app::App, storage::db::video::Video}; use anyhow::{Context, Result, bail}; use libmpv2::Mpv; @@ -19,22 +19,12 @@ use tokio::process::Command; use super::mpv_message; async fn run_self_in_external_command(app: &App, args: &[&str]) -> Result<()> { + // TODO(@bpeetz): Can we trust this value? <2025-06-15> let binary = env::current_exe().context("Failed to determine the current executable to re-execute")?; - let status = Command::new("riverctl") - .args(["focus-output", "next"]) - .status() - .await?; - if !status.success() { - bail!("focusing the next output failed!"); - } - let arguments = [ &[ - "--title", - "floating please", - "--command", binary .to_str() .context("Failed to turn the executable path to a utf8-string")?, @@ -49,29 +39,24 @@ async fn run_self_in_external_command(app: &App, args: &[&str]) -> Result<()> { ] .concat(); - let status = Command::new("alacritty").args(arguments).status().await?; - if !status.success() { - bail!("Falied to start `yt comments`"); - } - - let status = Command::new("riverctl") - .args(["focus-output", "next"]) + let status = Command::new(app.config.commands.external_spawn.first()) + .args(app.config.commands.external_spawn.tail()) + .args(arguments) .status() .await?; - if !status.success() { - bail!("focusing the next output failed!"); + bail!("Falied to start (external) `yt {}`", args.join(" ")); } Ok(()) } pub(super) async fn handle_yt_description_external(app: &App) -> Result<()> { - run_self_in_external_command(app, &["description"]).await?; + run_self_in_external_command(app, &["show", "description"]).await?; Ok(()) } pub(super) async fn handle_yt_description_local(app: &App, mpv: &Mpv) -> Result<()> { - let description: String = comments::description::get(app) + let description: String = Video::get_current_description(app) .await? .chars() .take(app.config.watch.local_displays_length) @@ -82,11 +67,11 @@ pub(super) async fn handle_yt_description_local(app: &App, mpv: &Mpv) -> Result< } pub(super) async fn handle_yt_comments_external(app: &App) -> Result<()> { - run_self_in_external_command(app, &["comments"]).await?; + run_self_in_external_command(app, &["show", "comments"]).await?; Ok(()) } pub(super) async fn handle_yt_comments_local(app: &App, mpv: &Mpv) -> Result<()> { - let comments: String = comments::get(app) + let comments: String = Video::get_current_comments(app) .await? .render(false) .chars() @@ -96,3 +81,13 @@ pub(super) async fn handle_yt_comments_local(app: &App, mpv: &Mpv) -> Result<()> mpv_message(mpv, &comments, Duration::from_secs(6))?; Ok(()) } + +pub(super) async fn handle_yt_info_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "info"]).await?; + Ok(()) +} + +pub(super) async fn handle_yt_thumbnail_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "thumbnail"]).await?; + Ok(()) +} diff --git a/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs b/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs new file mode 100644 index 0000000..bdb77d2 --- /dev/null +++ b/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs @@ -0,0 +1,225 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::time::Duration; + +use crate::{ + app::App, + storage::db::{ + insert::{Operations, playlist::VideoTransition}, + playlist::{Playlist, PlaylistIndex}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::{Context, Result}; +use libmpv2::{EndFileReason, Mpv, events::Event}; +use log::{debug, info}; + +mod client_messages; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum Status { + /// There are no videos cached and no more marked to be watched. + /// Waiting is pointless. + NoMoreAvailable, + + /// There are no videos cached, but some (> 0) are marked to be watched. + /// So we should wait for them to become available. + NoCached { marked_watch: usize }, + + /// There are videos cached and ready to be inserted into the playback queue. + Available { newly_available: usize }, +} + +fn mpv_message(mpv: &Mpv, message: &str, time: Duration) -> Result<()> { + mpv.command( + "show-text", + &[message, time.as_millis().to_string().as_str()], + )?; + Ok(()) +} + +/// Return the status of the playback queue +pub(crate) async fn status(app: &App) -> Result<Status> { + let playlist = Playlist::create(app).await?; + + let playlist_len = playlist.len(); + let marked_watch_num = Video::in_states(app, &[VideoStatusMarker::Watch]) + .await? + .len(); + + if playlist_len == 0 && marked_watch_num == 0 { + Ok(Status::NoMoreAvailable) + } else if playlist_len == 0 && marked_watch_num != 0 { + Ok(Status::NoCached { + marked_watch: marked_watch_num, + }) + } else if playlist_len != 0 { + Ok(Status::Available { + newly_available: playlist_len, + }) + } else { + unreachable!( + "The playlist length is {playlist_len}, but the number of marked watch videos is {marked_watch_num}! This is a bug." + ); + } +} + +/// # Returns +/// This will return [`true`], if the event handling should be stopped +/// +/// # Panics +/// Only if internal assertions fail. +#[allow(clippy::too_many_lines)] +pub(crate) async fn handle_mpv_event(app: &App, mpv: &Mpv, event: &Event<'_>) -> Result<bool> { + let mut ops = Operations::new("PlaylistHandler: handle event"); + + // Construct the playlist lazily. + // This avoids unneeded db lookups. + // (We use the moved `call_once` as guard for this) + let call_once = String::new(); + let playlist = move || { + drop(call_once); + Playlist::create(app) + }; + + let should_stop_event_handling = match event { + Event::EndFile(r) => match r.reason { + EndFileReason::Eof => { + info!("Mpv reached the end of the current video. Marking it watched."); + playlist().await?.resync_with_mpv(app, mpv)?; + + false + } + EndFileReason::Stop => { + // This reason is incredibly ambiguous. It _both_ means actually pausing a + // video and going to the next one in the playlist. + // Oh, and it's also called, when a video is removed from the playlist (at + // least via "playlist-remove current") + info!("Paused video (or went to next playlist entry); Doing nothing"); + + false + } + EndFileReason::Quit => { + info!("Mpv quit. Exiting playback"); + + playlist().await?.save_current_watch_progress(mpv, &mut ops); + + true + } + EndFileReason::Error => { + unreachable!("This should have been raised as a separate error") + } + EndFileReason::Redirect => { + // TODO: We probably need to handle this somehow <2025-02-17> + false + } + }, + Event::StartFile(_) => { + let mut playlist = playlist().await?; + + let mpv_pos = usize::try_from(mpv.get_property::<i64>("playlist-pos")?) + .expect("The value is strictly positive"); + + let yt_pos = playlist.current_index().map(usize::from); + + if (Some(mpv_pos) != yt_pos) || yt_pos.is_none() { + debug!( + "StartFileHandler: mpv pos {mpv_pos} and our pos {yt_pos:?} do not align. Reloading.." + ); + + if let Some((_, vid)) = playlist.get_focused_mut() { + vid.set_focused(false, &mut ops); + ops.commit(app) + .await + .context("Failed to commit video unfocusing")?; + + ops = Operations::new("PlaylistHandler: after set-focused"); + } + + let video = playlist + .get_mut(PlaylistIndex::from(mpv_pos)) + .expect("The mpv pos should not be out of bounds"); + + video.set_focused(true, &mut ops); + + playlist.resync_with_mpv(app, mpv)?; + } + + false + } + Event::Seek => { + playlist().await?.save_current_watch_progress(mpv, &mut ops); + + false + } + Event::ClientMessage(a) => { + debug!("Got Client Message event: '{}'", a.join(" ")); + + match a.as_slice() { + &["yt-comments-external"] => { + client_messages::handle_yt_comments_external(app).await?; + } + &["yt-comments-local"] => { + client_messages::handle_yt_comments_local(app, mpv).await?; + } + + &["yt-description-external"] => { + client_messages::handle_yt_description_external(app).await?; + } + &["yt-description-local"] => { + client_messages::handle_yt_description_local(app, mpv).await?; + } + + &["yt-info-external"] => { + client_messages::handle_yt_info_external(app).await?; + } + &["yt-thumbnail-external"] => { + client_messages::handle_yt_thumbnail_external(app).await?; + } + + &["yt-mark-picked"] => { + playlist().await?.mark_current_done( + app, + mpv, + VideoTransition::Picked, + &mut ops, + )?; + + mpv_message(mpv, "Marked the video as picked", Duration::from_secs(3))?; + } + &["yt-mark-watched"] => { + playlist().await?.mark_current_done( + app, + mpv, + VideoTransition::Watched, + &mut ops, + )?; + + mpv_message(mpv, "Marked the video watched", Duration::from_secs(3))?; + } + &["yt-check-new-videos"] => { + playlist().await?.resync_with_mpv(app, mpv)?; + } + other => { + debug!("Unknown message: {}", other.join(" ")); + } + } + + false + } + _ => false, + }; + + ops.commit(app).await?; + + Ok(should_stop_event_handling) +} diff --git a/crates/yt/src/commands/watch/mod.rs b/crates/yt/src/commands/watch/mod.rs new file mode 100644 index 0000000..ea4c513 --- /dev/null +++ b/crates/yt/src/commands/watch/mod.rs @@ -0,0 +1,24 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct WatchCommand { + /// Print the path to an ipc socket for mpv control to stdout at startup. + #[arg(long)] + provide_ipc_socket: bool, + + /// Don't start an mpv window at all. + #[arg(long)] + headless: bool, +} diff --git a/crates/yt/src/config/mod.rs b/crates/yt/src/config/mod.rs new file mode 100644 index 0000000..05bb4cf --- /dev/null +++ b/crates/yt/src/config/mod.rs @@ -0,0 +1,138 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use crate::config::support::mk_config; + +mod non_empty_vec; +mod paths; +mod support; + +pub(crate) static SHOULD_DISPLAY_COLOR: AtomicBool = AtomicBool::new(false); + +// We need to do both things to comply with what the config expects. +#[allow(clippy::trivially_copy_pass_by_ref, clippy::unnecessary_wraps)] +fn set_static_should_display_color(value: &bool) -> anyhow::Result<()> { + SHOULD_DISPLAY_COLOR.store(*value, Ordering::Relaxed); + + Ok(()) +} + +mk_config! { + use std::path::PathBuf; + use std::io::IsTerminal; + use std::time::Duration; + + use crate::shared::bytes::Bytes; + + use super::set_static_should_display_color; + + use super::paths::get_config_path; + use super::paths::get_runtime_path; + use super::paths::get_data_path; + use super::paths::ensure_parent_dir; + use super::paths::ensure_dir; + use super::paths::PREFIX; + + use super::non_empty_vec::NonEmptyVec; + use super::non_empty_vec::non_empty_vec; + + struct Config { + global: GlobalConfig = { + /// Whether to display colors. + display_colors: bool where display_color: Option<bool> =! {|config_value: Option<bool>| + Ok::<_, anyhow::Error>( + display_color + .unwrap_or( + config_value + .unwrap_or_else(|| std::io::stderr().is_terminal()) + ) + ) + } => set_static_should_display_color, + }, + select: SelectConfig = { + /// The playback speed to use, when it is not overridden. + playback_speed: f64 =: 2.7, + + /// The subtitle langs to download, when it is not overridden. + subtitle_langs: String =: String::new(), + }, + watch: WatchConfig = { + /// How many chars to display at most, when displaying information on mpv's local on screen + /// display. + local_displays_length: usize =: 1000, + + /// How long to wait between saving the video watch progress. + progress_save_intervall: Duration =: Duration::from_secs(10), + }, + commands: CommandsConfig = { + /// Which command to execute, when showing the thumbnail. + /// + /// This command will be executed with the one argument, being the path to the image file to display. + image_show: NonEmptyVec<String> =: non_empty_vec!["imv".to_owned()], + + /// Which command to use, when spawing one of the external commands (e.g. + /// `yt-comments-external` from mpv). + /// + /// The command will be called with a series of args that should be executed. + /// For example, + /// `<your_specified_command> <path_to_yt_binary> --db-path <path_to_current_db_path> comments` + external_spawn: NonEmptyVec<String> =: non_empty_vec!["alacritty".to_owned(), "-e".to_owned()], + + /// Which command to use, when opening video urls (like in the `yt select url` case). + /// + /// This command will be called with one argument, being the url of the video to open. + url_opener: NonEmptyVec<String> =: non_empty_vec!["firefox".to_owned()], + }, + paths: PathsConfig = { + /// Where to store downloaded files. + download_dir: PathBuf =: { + // We download to the temp dir to avoid taxing the disk + let temp_dir = std::env::temp_dir(); + + temp_dir.join(PREFIX) + } => ensure_dir, + + /// Path to the mpv configuration file. + mpv_config_path: PathBuf =? get_config_path("mpv.conf") => ensure_parent_dir, + + /// Path to the mpv input configuration file. + mpv_input_path: PathBuf =? get_config_path("mpv.input.conf") => ensure_parent_dir, + + /// Which path to use for mpv ipc socket creation. + mpv_ipc_socket_path: PathBuf =? get_runtime_path("mpv.ipc.socket") => ensure_parent_dir, + + /// Path to the video database. + database_path: PathBuf where db_path: Option<PathBuf> =! {|config_value: Option<PathBuf>| { + db_path.map_or_else(|| config_value.map_or_else(|| get_data_path("videos.sqlite"), Ok), Ok) + }} => ensure_parent_dir, + + /// Where to store the selection file before applying it. + last_selection_path: PathBuf =? get_runtime_path("selected.yts") => ensure_parent_dir, + }, + download: DownloadConfig = { + /// The maximum cache size. + max_cache_size: Bytes =? "3 GiB".parse(), + }, + update: UpdateConfig = { + /// How many videos to download, when checking for new ones. + max_backlog: usize =: 20, + + /// How many threads to use in the thread pool for fetching new videos. + pool_size: usize =: 16, + + /// How many subscriptions to fetch at once. + /// + /// For example, 16 means, that we will fetch 16 subscriptions at the same time. + futures: usize =: 16 * 4, + }, + } +} diff --git a/crates/yt/src/config/non_empty_vec.rs b/crates/yt/src/config/non_empty_vec.rs new file mode 100644 index 0000000..bd2c5e3 --- /dev/null +++ b/crates/yt/src/config/non_empty_vec.rs @@ -0,0 +1,83 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::VecDeque, + fmt::{Display, Write}, +}; + +use anyhow::bail; +use serde::{Deserialize, Serialize}; + +macro_rules! non_empty_vec { + ($first:expr $(, $($others:expr),+ $(,)?)?) => {{ + let inner: Vec<_> = vec![$first $(, $($others,)+)?]; + inner.try_into().expect("Has a first arg") + }} +} +pub(crate) use non_empty_vec; + +/// A vector that is non-empty. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(try_from = "Vec<T>")] +#[serde(into = "Vec<T>")] +pub(crate) struct NonEmptyVec<T: Clone> { + first: T, + rest: Vec<T>, +} + +impl<T: Clone> TryFrom<Vec<T>> for NonEmptyVec<T> { + type Error = anyhow::Error; + + fn try_from(value: Vec<T>) -> Result<Self, Self::Error> { + let mut queue = VecDeque::from(value); + + if let Some(first) = queue.pop_front() { + Ok(Self { + first, + rest: queue.into(), + }) + } else { + bail!("You need to have at least one element in a non-empty vector.") + } + } +} + +impl<T: Clone> From<NonEmptyVec<T>> for Vec<T> { + fn from(value: NonEmptyVec<T>) -> Self { + let mut base = vec![value.first]; + base.extend(value.rest); + base + } +} + +impl<T: Clone> NonEmptyVec<T> { + pub(crate) fn first(&self) -> &T { + &self.first + } + + pub(crate) fn tail(&self) -> &[T] { + self.rest.as_ref() + } + + pub(crate) fn join(&self, sep: &str) -> String + where + T: Display, + { + let mut output = String::new(); + write!(output, "{}", self.first()).expect("In-memory, does not fail"); + + for elem in &self.rest { + write!(output, "{sep}{elem}").expect("In-memory, does not fail"); + } + + output + } +} diff --git a/crates/yt/src/config/paths.rs b/crates/yt/src/config/paths.rs new file mode 100644 index 0000000..66975dd --- /dev/null +++ b/crates/yt/src/config/paths.rs @@ -0,0 +1,58 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +pub(super) fn get_runtime_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_runtime_file(name) + .with_context(|| format!("Failed to place runtime file: '{name}'")) +} +pub(super) fn get_data_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_data_file(name) + .with_context(|| format!("Failed to place data file: '{name}'")) +} +pub(super) fn get_config_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_config_file(name) + .with_context(|| format!("Failed to place config file: '{name}'")) +} + +pub(super) fn ensure_parent_dir(path: &Path) -> Result<()> { + if !path.exists() { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + } + + Ok(()) +} +pub(super) fn ensure_dir(path: &Path) -> Result<()> { + if !path.exists() { + std::fs::create_dir_all(path) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + + Ok(()) +} + +pub(super) fn config_path() -> Result<PathBuf> { + get_config_path("config.toml") +} + +pub(crate) const PREFIX: &str = "yt"; diff --git a/crates/yt/src/config/support.rs b/crates/yt/src/config/support.rs new file mode 100644 index 0000000..96e7ba4 --- /dev/null +++ b/crates/yt/src/config/support.rs @@ -0,0 +1,161 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +macro_rules! mk_config { + ( + $(use $usage_path:path;)* + + struct $name:ident { + $( + $(#[$attr0:meta])* + $subconfig_name:ident : $subconfig_type:ident = { + $( + $(#[$attr1:meta])* + $field_name:ident : $field_type:ty $( + where $extra_input:ident: $extra_input_type:ty + ),* = $errors:tt $default:expr $(=> $finalizer:ident)? + ),* + $(,)? + } + ),* + $(,)? + } + ) => { + mod _inner { + #![allow(non_snake_case)] + + $(use $usage_path;)* + + #[derive(serde::Serialize, Debug)] + pub(crate) struct $name { + $( + $(#[$attr0])* + pub(crate) $subconfig_name: $subconfig_type + ),* + } + + #[derive(Debug, serde::Deserialize, PartialEq)] + #[serde(deny_unknown_fields)] + #[allow(non_camel_case_types)] + struct config { + $( + $subconfig_name: Option<$subconfig_name> + ),* + } + + impl $name { + pub(crate) fn from_config_file( + config_file_path: Option<std::path::PathBuf>, + $( + $( + $( + $extra_input: $extra_input_type, + )* + )* + )* + ) -> anyhow::Result<Self> { + use anyhow::Context; + + let config_file_path = + config_file_path.map_or_else(|| -> anyhow::Result<_> { super::paths::config_path() }, Ok)?; + + let config: config = + toml::from_str(&std::fs::read_to_string(config_file_path).unwrap_or(String::new())) + .context("Failed to parse the config file as toml")?; + + Ok(Self { + $( + $subconfig_name: { + let toplevel = config.$subconfig_name.unwrap_or_default(); + $subconfig_type { + $( + $field_name: $field_name(toplevel.$field_name, $($extra_input),*)? + ),* + } + } + ),* + }) + } + + pub(crate) fn run_finalizers(&self) -> anyhow::Result<()> { + #[allow(unused_imports)] + use anyhow::Context; + + $( + $( + $( + $finalizer(&self.$subconfig_name.$field_name) + .context( + concat!( + "While running the finalizer for config value '", + stringify!($subconfig_name), + ".", + stringify!($field_name), + "'" + ) + )?; + )? + )* + )* + + Ok(()) + } + } + + $( + #[derive(serde::Serialize, Debug)] + pub(crate) struct $subconfig_type { + $( + $(#[$attr1])* + pub(crate) $field_name: $field_type + ),* + } + + #[derive(Debug, Default, serde::Deserialize, PartialEq)] + #[serde(deny_unknown_fields)] + #[allow(non_camel_case_types)] + struct $subconfig_name { + $( + $field_name: Option<$field_type> + ),* + } + + $( + fn $field_name( + config_value: Option<$field_type>, + $($extra_input: $extra_input_type),* + ) -> anyhow::Result<$field_type> { + use anyhow::Context; + + let expr = $crate::config::support::maybe_wrap_type!($field_type =$errors $default)(config_value); + + expr.context(concat!("Failed to generate default config value for '", stringify!($field_name),"'")) + } + )* + )* + } + pub(crate) use self::_inner::*; + }; +} + +macro_rules! maybe_wrap_type { + ($ty:ty =! $val:expr) => { + (|config_value: Option<$ty>| $val(config_value)) + }; + ($ty:ty =? $val:expr) => { + (|config_value: Option<$ty>| config_value.map_or_else(|| $val, Ok)) + }; + ($ty:ty =: $val:expr) => { + (|config_value: Option<$ty>| Ok::<_, anyhow::Error>(config_value.unwrap_or_else(|| $val))) + }; +} + +pub(crate) use maybe_wrap_type; +pub(crate) use mk_config; diff --git a/crates/yt/src/main.rs b/crates/yt/src/main.rs new file mode 100644 index 0000000..705e642 --- /dev/null +++ b/crates/yt/src/main.rs @@ -0,0 +1,89 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// `yt` is not a library. Besides, the `anyhow::Result` type is really useless, if you're not going +// to print it anyways. +#![allow(clippy::missing_errors_doc, clippy::missing_panics_doc)] + +use anyhow::{Context, Result}; +use app::App; +use clap::{CommandFactory, Parser}; +use config::Config; +use log::info; + +use crate::commands::Command; + +pub(crate) mod output; +pub(crate) mod yt_dlp; + +pub(crate) mod ansi_escape_codes; +pub(crate) mod app; +pub(crate) mod cli; +pub(crate) mod commands; +pub(crate) mod shared; + +pub(crate) mod config; +pub(crate) mod select; +pub(crate) mod storage; +pub(crate) mod version; +pub(crate) mod videos; + +#[tokio::main] +async fn main() -> Result<()> { + clap_complete::CompleteEnv::with_factory(cli::CliArgs::command).complete(); + + let args = cli::CliArgs::parse(); + + // The default verbosity is 1 (Warn) + let verbosity: u8 = args.verbosity + 1; + + stderrlog::new() + .module(module_path!()) + .modules(&["yt_dlp".to_owned(), "libmpv2".to_owned()]) + .quiet(args.quiet) + .show_module_names(false) + .color(stderrlog::ColorChoice::Auto) + .verbosity(verbosity as usize) + .timestamp(stderrlog::Timestamp::Off) + .init() + .expect("Let's just hope that this does not panic"); + + info!("Using verbosity level: '{} ({})'", verbosity, { + match verbosity { + 0 => "Error", + 1 => "Warn", + 2 => "Info", + 3 => "Debug", + 4.. => "Trace", + } + }); + + let config = Config::from_config_file(args.config_path, args.color, args.db_path)?; + if args.version { + version::show(&config).await?; + return Ok(()); + } + + // Perform config finalization _after_ checking for the version + // so that version always works. + config + .run_finalizers() + .context("Failed to finalize config for usage")?; + + let app = App::new(config, !args.no_migrate_db).await?; + + args.command + .unwrap_or(Command::default()) + .implm(app) + .await?; + + Ok(()) +} diff --git a/yt/src/comments/output.rs b/crates/yt/src/output/mod.rs index cb3a9c4..2f74519 100644 --- a/yt/src/comments/output.rs +++ b/crates/yt/src/output/mod.rs @@ -17,9 +17,7 @@ use std::{ use anyhow::{Context, Result}; use uu_fmt::{FmtOptions, process_text}; -use crate::unreachable::Unreachable; - -pub async fn display_fmt_and_less(input: String) -> Result<()> { +pub(crate) fn display_less(input: String) -> Result<()> { let mut less = Command::new("less") .args(["--raw-control-chars"]) .stdin(Stdio::piped()) @@ -27,12 +25,11 @@ pub async fn display_fmt_and_less(input: String) -> Result<()> { .spawn() .context("Failed to run less")?; - let input = format_text(&input); let mut stdin = less.stdin.take().context("Failed to open stdin")?; std::thread::spawn(move || { stdin .write_all(input.as_bytes()) - .unreachable("Should be able to write to the stdin of less"); + .expect("Should be able to write to the stdin of less"); }); let _ = less.wait().context("Failed to await less")?; @@ -40,9 +37,15 @@ pub async fn display_fmt_and_less(input: String) -> Result<()> { Ok(()) } +pub(crate) fn display_fmt_and_less(input: &str) -> Result<()> { + display_less(format_text(&input, None)) +} + #[must_use] -pub fn format_text(input: &str) -> String { - let width = termsize::get().map_or(90, |size| size.cols); +pub(crate) fn format_text(input: &str, termsize: Option<u16>) -> String { + let input = input.trim(); + + let width = termsize.unwrap_or_else(|| termsize::get().map_or(90, |size| size.cols)); let fmt_opts = FmtOptions { uniform: true, split_only: true, diff --git a/crates/yt/src/select/duration.rs b/crates/yt/src/select/duration.rs new file mode 100644 index 0000000..f1de2ea --- /dev/null +++ b/crates/yt/src/select/duration.rs @@ -0,0 +1,240 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::str::FromStr; +use std::time::Duration; + +use anyhow::{Result, bail}; + +const SECOND: u64 = 1; +const MINUTE: u64 = 60 * SECOND; +const HOUR: u64 = 60 * MINUTE; +const DAY: u64 = 24 * HOUR; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct MaybeDuration { + time: Option<Duration>, +} + +impl MaybeDuration { + #[must_use] + pub(crate) fn from_std(d: Duration) -> Self { + Self { time: Some(d) } + } + + #[must_use] + pub(crate) fn from_secs_f64(d: f64) -> Self { + Self { + time: Some(Duration::from_secs_f64(d)), + } + } + #[must_use] + pub(crate) fn from_maybe_secs_f64(d: Option<f64>) -> Self { + Self { + time: d.map(Duration::from_secs_f64), + } + } + #[must_use] + #[cfg(test)] + pub(crate) fn from_secs(d: u64) -> Self { + Self { + time: Some(Duration::from_secs(d)), + } + } + + /// Try to return the current duration encoded as seconds. + #[must_use] + pub(crate) fn as_secs(&self) -> Option<u64> { + self.time.map(|v| v.as_secs()) + } + + /// Try to return the current duration encoded as seconds and nanoseconds. + #[must_use] + pub(crate) fn as_secs_f64(&self) -> Option<f64> { + self.time.map(|v| v.as_secs_f64()) + } +} + +impl FromStr for MaybeDuration { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + #[derive(Debug, Clone, Copy)] + enum Token { + Number(u64), + UnitConstant((char, u64)), + } + + struct Tokenizer<'a> { + input: &'a str, + } + + impl Tokenizer<'_> { + fn next(&mut self) -> Result<Option<Token>> { + loop { + if let Some(next) = self.peek() { + match next { + '0'..='9' => { + let mut number = self.expect_num(); + while matches!(self.peek(), Some('0'..='9')) { + number *= 10; + number += self.expect_num(); + } + break Ok(Some(Token::Number(number))); + } + 's' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('s', SECOND)))); + } + 'm' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('m', MINUTE)))); + } + 'h' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('h', HOUR)))); + } + 'd' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('d', DAY)))); + } + ' ' => { + // Simply ignore white space + self.chomp(); + } + other => bail!("Unknown unit: {other:#?}"), + } + } else { + break Ok(None); + } + } + } + + fn chomp(&mut self) { + self.input = &self.input[1..]; + } + + fn peek(&self) -> Option<char> { + self.input.chars().next() + } + + fn expect_num(&mut self) -> u64 { + let next = self.peek().expect("Should be some at this point"); + self.chomp(); + assert!(next.is_ascii_digit()); + (next as u64) - ('0' as u64) + } + } + + if s == "[No duration]" { + return Ok(Self { time: None }); + } + + let mut tokenizer = Tokenizer { input: s }; + + let mut value = 0; + let mut current_val = None; + while let Some(token) = tokenizer.next()? { + match token { + Token::Number(number) => { + if let Some(current_val) = current_val { + bail!("Failed to find unit for number: {current_val}"); + } + + { + current_val = Some(number); + } + } + Token::UnitConstant((name, unit)) => { + if let Some(cval) = current_val { + value += cval * unit; + current_val = None; + } else { + bail!("Found unit without number: {name:#?}"); + } + } + } + } + + if let Some(current_val) = current_val { + bail!("Duration endet without unit, number was: {current_val}"); + } + + Ok(Self { + time: Some(Duration::from_secs(value)), + }) + } +} + +impl std::fmt::Display for MaybeDuration { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + if let Some(self_seconds) = self.as_secs() { + let base_day = self_seconds - (self_seconds % DAY); + let base_hour = (self_seconds % DAY) - ((self_seconds % DAY) % HOUR); + let base_min = (self_seconds % HOUR) - (((self_seconds % DAY) % HOUR) % MINUTE); + let base_sec = ((self_seconds % DAY) % HOUR) % MINUTE; + + let d = base_day / DAY; + let h = base_hour / HOUR; + let m = base_min / MINUTE; + let s = base_sec / SECOND; + + if d > 0 { + write!(fmt, "{d}d {h}h {m}m") + } else if h > 0 { + write!(fmt, "{h}h {m}m") + } else { + write!(fmt, "{m}m {s}s") + } + } else { + write!(fmt, "[No duration]") + } + } +} +#[cfg(test)] +mod test { + use std::str::FromStr; + + use crate::select::duration::{DAY, HOUR, MINUTE}; + + use super::MaybeDuration; + + fn mk_roundtrip(input: MaybeDuration, expected: &str) { + let output = MaybeDuration::from_str(expected).unwrap(); + + assert_eq!(input.to_string(), output.to_string()); + assert_eq!(input.to_string(), expected); + assert_eq!( + MaybeDuration::from_str(input.to_string().as_str()).unwrap(), + output + ); + } + + #[test] + fn test_roundtrip_duration_1h() { + mk_roundtrip(MaybeDuration::from_secs(HOUR), "1h 0m"); + } + #[test] + fn test_roundtrip_duration_30min() { + mk_roundtrip(MaybeDuration::from_secs(MINUTE * 30), "30m 0s"); + } + #[test] + fn test_roundtrip_duration_1d() { + mk_roundtrip( + MaybeDuration::from_secs(DAY + MINUTE * 30 + HOUR * 2), + "1d 2h 30m", + ); + } + #[test] + fn test_roundtrip_duration_none() { + mk_roundtrip(MaybeDuration::from_maybe_secs_f64(None), "[No duration]"); + } +} diff --git a/crates/yt/src/select/mod.rs b/crates/yt/src/select/mod.rs new file mode 100644 index 0000000..b02677f --- /dev/null +++ b/crates/yt/src/select/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod duration; + +// // FIXME: There should be no reason why we need to re-run yt, just to get the help string. But I've +// // yet to find a way to do it without the extra exec <2024-08-20> +// async fn get_help() -> Result<String> { +// let binary_name = current_exe()?; +// let cmd = Command::new(binary_name) +// .args(&["select", "--help"]) +// .output() +// .await?; +// +// assert_eq!(cmd.status.code(), Some(0)); +// +// let output = String::from_utf8(cmd.stdout).expect("Our help output was not utf8?"); +// +// let out = output +// .lines() +// .map(|line| format!("# {}\n", line)) +// .collect::<String>(); +// +// debug!("Returning help: '{}'", &out); +// +// Ok(out) +// } diff --git a/crates/bytes/src/error.rs b/crates/yt/src/shared/bytes/error.rs index c9783d8..c9783d8 100644 --- a/crates/bytes/src/error.rs +++ b/crates/yt/src/shared/bytes/error.rs diff --git a/crates/bytes/src/lib.rs b/crates/yt/src/shared/bytes/mod.rs index 2a9248d..31e782e 100644 --- a/crates/bytes/src/lib.rs +++ b/crates/yt/src/shared/bytes/mod.rs @@ -16,6 +16,7 @@ )] use std::{fmt::Display, str::FromStr}; +use ::serde::{Deserialize, Serialize}; use error::BytesError; const B: u64 = 1; @@ -31,10 +32,11 @@ const MB: u64 = 1000 * KB; const GB: u64 = 1000 * MB; const TB: u64 = 1000 * GB; -pub mod error; -pub mod serde; +pub(crate) mod error; -#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq, Deserialize, Serialize)] +#[serde(try_from = "String")] +#[serde(into = "String")] pub struct Bytes(u64); impl Bytes { @@ -131,6 +133,20 @@ impl Display for Bytes { } } +impl From<Bytes> for String { + fn from(value: Bytes) -> Self { + value.to_string() + } +} + +impl TryFrom<String> for Bytes { + type Error = BytesError; + + fn try_from(value: String) -> Result<Self, Self::Error> { + value.as_str().parse() + } +} + // taken from this stack overflow question: https://stackoverflow.com/a/76572321 /// Round to significant digits (rather than digits after the decimal). /// @@ -149,7 +165,7 @@ impl Display for Bytes { ///# } /// ``` #[must_use] -pub fn precision_f64(x: f64, decimals: u32) -> f64 { +pub(crate) fn precision_f64(x: f64, decimals: u32) -> f64 { if x == 0. || decimals == 0 { 0. } else { diff --git a/crates/yt_dlp/src/wrapper/mod.rs b/crates/yt/src/shared/mod.rs index 3fe3247..d3cc563 100644 --- a/crates/yt_dlp/src/wrapper/mod.rs +++ b/crates/yt/src/shared/mod.rs @@ -1,6 +1,6 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // // This file is part of Yt. @@ -8,5 +8,4 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -pub mod info_json; -// pub mod yt_dlp_options; +pub(crate) mod bytes; diff --git a/crates/yt/src/storage/db/extractor_hash.rs b/crates/yt/src/storage/db/extractor_hash.rs new file mode 100644 index 0000000..3ad8273 --- /dev/null +++ b/crates/yt/src/storage/db/extractor_hash.rs @@ -0,0 +1,220 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashSet, fmt::Display, str::FromStr}; + +use anyhow::{Context, Result, bail}; +use blake3::Hash; +use log::debug; +use serde::{Deserialize, Serialize}; +use tokio::sync::OnceCell; +use yt_dlp::{info_json::InfoJson, json_cast, json_get, json_try_get}; + +use crate::app::App; + +static EXTRACTOR_HASH_LENGTH: OnceCell<usize> = OnceCell::const_new(); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, Serialize, Deserialize)] +pub(crate) struct ExtractorHash { + hash: Hash, +} + +impl Display for ExtractorHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.hash.fmt(f) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ShortHash(String); + +impl Display for ShortHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone)] +#[allow(clippy::module_name_repetitions)] +pub(crate) struct LazyExtractorHash { + value: ShortHash, +} + +impl FromStr for LazyExtractorHash { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { + // perform some cheap validation + if s.len() > 64 { + bail!("A hash can only contain 64 bytes!"); + } + + Ok(Self { + value: ShortHash(s.to_owned()), + }) + } +} + +impl LazyExtractorHash { + /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] + pub(crate) async fn realize( + self, + app: &App, + all_hashes: Option<&[ExtractorHash]>, + ) -> Result<ExtractorHash> { + ExtractorHash::from_short_hash(app, &self.value, all_hashes).await + } +} + +impl ExtractorHash { + #[must_use] + pub(crate) fn from_hash(hash: Hash) -> Self { + Self { hash } + } + + pub(crate) async fn from_short_hash( + app: &App, + s: &ShortHash, + all_hashes: Option<&[Self]>, + ) -> Result<Self> { + let all_hashes = if let Some(all) = all_hashes { + all + } else { + &Self::get_all(app) + .await + .context("Failed to fetch all extractor-hashes from the database")? + }; + let needed_chars = s.0.len(); + for hash in all_hashes { + // PERFORMANCE(@bpeetz): This could avoid the string construction and just use a + // numeric equality check instead. <2025-07-15> + if hash.hash().to_hex()[..needed_chars] == s.0 { + return Ok(*hash); + } + } + bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); + } + + pub(crate) fn from_info_json(entry: &InfoJson) -> Self { + // HACK(@bpeetz): The code that follows is a gross hack. + // One would expect the `id` to be unique _and_ constant for each and every possible info JSON. + // But .. it's just not. The `ARDMediathek` extractor, will sometimes return different `id`s for the same + // video, effectively causing us to insert the same video again into the db (which fails, + // because the URL is still unique). + // + // As such we _should_ probably find a constant value for all extractors, but that just does + // not exist currently, without processing each entry (which is expensive and which I would + // like to avoid). + // + // Therefor, we simply special case the `ARDBetaMediathek` extractor. <2025-07-04> + + // NOTE(@bpeetz): `yt-dlp` apparently uses these two different names for the same thing <2025-07-04> + let ie_key = { + if let Some(ie_key) = json_try_get!(entry, "ie_key", as_str) { + ie_key + } else if let Some(extractor_key) = json_try_get!(entry, "extractor_key", as_str) { + extractor_key + } else { + unreachable!( + "Either `ie_key` or `extractor_key` \ + should be set on every entry info json" + ) + } + }; + + if ie_key == "ARDBetaMediathek" { + // NOTE(@bpeetz): The mediathek is changing their Id scheme, from an `short` old Id to the + // new id. As the new id is too long for some people, yt-dlp will be default return the old + // one (when it is still available!). The new one is called `display_id`. + // Therefore, we simply check if the new one is explicitly returned, and otherwise use the + // normal `id` value, as these are cases where the old one is no longer available. <2025-07-04> + let id = if let Some(display_id) = json_try_get!(entry, "display_id", as_str) { + display_id.as_bytes() + } else { + json_get!(entry, "id", as_str).as_bytes() + }; + + Self { + hash: blake3::hash(id), + } + } else { + Self { + hash: blake3::hash(json_get!(entry, "id", as_str).as_bytes()), + } + } + } + + #[must_use] + pub(crate) fn hash(&self) -> &Hash { + &self.hash + } + + pub(crate) async fn as_short_hash(&self, app: &App) -> Result<ShortHash> { + let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { + *needed_chars + } else { + let needed_chars = self + .get_needed_char_len(app) + .await + .context("Failed to calculate needed char length")?; + EXTRACTOR_HASH_LENGTH + .set(needed_chars) + .expect("This should work at this stage, as we checked above that it is empty."); + + needed_chars + }; + + Ok(ShortHash( + self.hash() + .to_hex() + .chars() + .take(needed_chars) + .collect::<String>(), + )) + } + + async fn get_needed_char_len(&self, app: &App) -> Result<usize> { + debug!("Calculating the needed hash char length"); + let all_hashes = Self::get_all(app) + .await + .context("Failed to fetch all extractor -hashesh from database")?; + + let all_char_vec_hashes = all_hashes + .into_iter() + .map(|hash| hash.hash().to_hex().chars().collect::<Vec<char>>()) + .collect::<Vec<Vec<_>>>(); + + // This value should be updated later, if not rust will panic in the assertion. + let mut needed_chars: usize = 1000; + 'outer: for i in 1..64 { + let i_chars: Vec<String> = all_char_vec_hashes + .iter() + .map(|vec| vec.iter().take(i).collect::<String>()) + .collect(); + + let mut uniqnes_hashmap: HashSet<String> = HashSet::new(); + for ch in i_chars { + if !uniqnes_hashmap.insert(ch) { + // The key was already in the hash map, thus we have a duplicated char and need + // at least one char more + continue 'outer; + } + } + + needed_chars = i; + break 'outer; + } + + assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); + + Ok(needed_chars) + } +} diff --git a/crates/yt/src/storage/db/get/extractor_hash.rs b/crates/yt/src/storage/db/get/extractor_hash.rs new file mode 100644 index 0000000..c8e150a --- /dev/null +++ b/crates/yt/src/storage/db/get/extractor_hash.rs @@ -0,0 +1,68 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use blake3::Hash; +use sqlx::{SqliteConnection, query}; + +use crate::{ + app::App, + storage::db::{ + extractor_hash::ExtractorHash, + video::{Video, video_from_record}, + }, +}; + +impl ExtractorHash { + pub(crate) async fn get(&self, txn: &mut SqliteConnection) -> Result<Video> { + let extractor_hash = self.hash().to_string(); + + let base = query!( + r#" + SELECT * + FROM videos + WHERE extractor_hash = ? + "#, + extractor_hash + ) + .fetch_one(txn) + .await?; + + Ok(video_from_record!(base)) + } + + pub(crate) async fn get_with_app(&self, app: &App) -> Result<Video> { + let mut txn = app.database.begin().await?; + let out = self.get(&mut txn).await?; + txn.commit().await?; + + Ok(out) + } + + pub(crate) async fn get_all(app: &App) -> Result<Vec<Self>> { + let hashes_hex = query!( + r#" + SELECT extractor_hash + FROM videos; + "# + ) + .fetch_all(&app.database) + .await?; + + Ok(hashes_hex + .iter() + .map(|hash| { + Self::from_hash(Hash::from_hex(&hash.extractor_hash).expect( + "These values started as blake3 hashes, they should stay blake3 hashes", + )) + }) + .collect()) + } +} diff --git a/crates/yt/src/storage/db/get/mod.rs b/crates/yt/src/storage/db/get/mod.rs new file mode 100644 index 0000000..4bcd066 --- /dev/null +++ b/crates/yt/src/storage/db/get/mod.rs @@ -0,0 +1,15 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod extractor_hash; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod txn_log; +pub(crate) mod video; diff --git a/crates/yt/src/storage/db/get/playlist.rs b/crates/yt/src/storage/db/get/playlist.rs new file mode 100644 index 0000000..5094523 --- /dev/null +++ b/crates/yt/src/storage/db/get/playlist.rs @@ -0,0 +1,68 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{ + playlist::{Playlist, PlaylistIndex}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +impl Playlist { + /// Get an video based in its index. + #[must_use] + pub(crate) fn get_mut(&mut self, index: PlaylistIndex) -> Option<&mut Video> { + self.videos.get_mut(Into::<usize>::into(index)) + } + + /// Create a playlist, by loading it from the database. + pub(crate) async fn create(app: &App) -> Result<Self> { + let videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + Ok(Self { videos }) + } + + /// Return the current playlist index. + /// + /// This effectively looks for the currently focused video and returns it's index. + /// + /// # Panics + /// Only if internal assertions fail. + pub(crate) fn current_index(&self) -> Option<PlaylistIndex> { + if let Some((index, _)) = self.get_focused() { + Some(index) + } else { + None + } + } + + /// Get the currently focused video, if it exists. + #[must_use] + pub(crate) fn get_focused_mut(&mut self) -> Option<(PlaylistIndex, &mut Video)> { + self.videos + .iter_mut() + .enumerate() + .find(|(_, v)| v.is_focused()) + .map(|(index, video)| (PlaylistIndex::from(index), video)) + } + + /// Get the currently focused video, if it exists. + #[must_use] + pub(crate) fn get_focused(&self) -> Option<(PlaylistIndex, &Video)> { + self.videos + .iter() + .enumerate() + .find(|(_, v)| v.is_focused()) + .map(|(index, video)| (PlaylistIndex::from(index), video)) + } +} diff --git a/crates/yt/src/storage/db/get/subscription.rs b/crates/yt/src/storage/db/get/subscription.rs new file mode 100644 index 0000000..16a6e8b --- /dev/null +++ b/crates/yt/src/storage/db/get/subscription.rs @@ -0,0 +1,49 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::collections::HashMap; + +use crate::{ + app::App, + storage::db::subscription::{Subscription, Subscriptions}, +}; + +use anyhow::Result; +use sqlx::query; +use url::Url; + +impl Subscriptions { + /// Get a list of subscriptions + pub(crate) async fn get(app: &App) -> Result<Self> { + let raw_subs = query!( + " + SELECT * + FROM subscriptions; + " + ) + .fetch_all(&app.database) + .await?; + + let subscriptions: HashMap<String, Subscription> = raw_subs + .into_iter() + .map(|sub| { + ( + sub.name.clone(), + Subscription::new( + sub.name, + Url::parse(&sub.url).expect("It was an URL, when we inserted it."), + ), + ) + }) + .collect(); + + Ok(Subscriptions(subscriptions)) + } +} diff --git a/crates/yt/src/storage/db/get/txn_log.rs b/crates/yt/src/storage/db/get/txn_log.rs new file mode 100644 index 0000000..1a6df2c --- /dev/null +++ b/crates/yt/src/storage/db/get/txn_log.rs @@ -0,0 +1,43 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{insert::Committable, txn_log::TxnLog, video::TimeStamp}, +}; + +use anyhow::Result; +use sqlx::query; + +impl<O: Committable> TxnLog<O> { + /// Get the log of all operations that have been performed. + pub(crate) async fn get(app: &App) -> Result<Self> { + let raw_ops = query!( + " + SELECT * + FROM txn_log + ORDER BY timestamp ASC; + " + ) + .fetch_all(&app.database) + .await?; + + let inner = raw_ops + .into_iter() + .filter_map(|raw_op| { + serde_json::from_str(&raw_op.operation) + .map(|parsed_op| (TimeStamp::from_secs(raw_op.timestamp), parsed_op)) + .ok() + }) + .collect(); + + Ok(TxnLog::new(inner)) + } +} diff --git a/crates/yt/src/storage/db/get/video/mod.rs b/crates/yt/src/storage/db/get/video/mod.rs new file mode 100644 index 0000000..69adb6b --- /dev/null +++ b/crates/yt/src/storage/db/get/video/mod.rs @@ -0,0 +1,261 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{fs::File, path::PathBuf}; + +use anyhow::{Context, Result, bail}; +use log::debug; +use sqlx::query; +use yt_dlp::{info_json::InfoJson, json_cast, json_try_get}; + +use crate::{ + app::App, + storage::db::video::{ + Video, VideoStatus, VideoStatusMarker, + comments::{Comments, raw::RawComment}, + video_from_record, + }, +}; + +impl Video { + /// Returns to next video which should be downloaded. This respects the priority assigned by select. + /// It does not return videos, which are already downloaded. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) async fn next_to_download(app: &App) -> Result<Option<Self>> { + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + // NOTE: The ORDER BY statement should be the same as the one in [`in_states`]. <2024-08-22> + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND cache_path IS NULL + ORDER BY priority DESC, publish_date DESC + LIMIT 1; + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record!(base))) + } + } + + /// Returns the description of the current video. + /// The returned description will be set to `<No description>` in the absence of one. + /// + /// # Errors + /// If no current video exists. + /// + /// # Panics + /// If the current video lacks the `info.json` file. + pub(crate) async fn get_current_description(app: &App) -> Result<String> { + let Some(currently_playing_video) = Video::currently_focused(app).await? else { + bail!("Could not find a currently playing video!"); + }; + + let info_json = ¤tly_playing_video.get_info_json()?.expect( + "A currently *playing* must be cached. \ + And thus the info.json should be available.", + ); + + let description = json_try_get!(info_json, "description", as_str) + .unwrap_or("<No description>") + .to_owned(); + + Ok(description) + } + + /// Returns the comments of the current video. + /// The returned [`Comments`] will be empty in the absence of comments. + /// + /// # Errors + /// If no current video exists. + /// + /// # Panics + /// If the current video lacks the `info.json` file. + pub(crate) async fn get_current_comments(app: &App) -> Result<Comments> { + let Some(currently_playing_video) = Video::currently_focused(app).await? else { + bail!("Could not find a currently playing video!"); + }; + + let info_json = ¤tly_playing_video.get_info_json()?.expect( + "A currently *playing* video must be cached. \ + And thus the info.json should be available.", + ); + + let raw_comments = if let Some(comments) = json_try_get!(info_json, "comments", as_array) { + comments + .iter() + .cloned() + .map(serde_json::from_value) + .collect::<Result<Vec<RawComment>, _>>()? + } else { + // TODO(@bpeetz): We could display a `<No-comments>` here. <2025-07-15> + + bail!( + "The video ('{}') does not have comments!", + json_try_get!(info_json, "title", as_str).unwrap_or("<No Title>") + ) + }; + + Ok(Comments::from_raw(raw_comments)) + } + + /// Optionally returns the video that is currently focused. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) async fn currently_focused(app: &App) -> Result<Option<Self>> { + let status = VideoStatusMarker::Cached.as_db_integer(); + + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND is_focused = 1 + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record!(base))) + } + } + + /// Calculate the [`info_json`] location on-disk for this video. + /// + /// Will return [`None`], if the video does not have an downloaded [`info_json`] + pub(crate) fn info_json_path(&self) -> Result<Option<PathBuf>> { + if let VideoStatus::Cached { mut cache_path, .. } = self.status.clone() { + if !cache_path.set_extension("info.json") { + bail!( + "Failed to change path extension to 'info.json': {}", + cache_path.display() + ); + } + + Ok(Some(cache_path)) + } else { + Ok(None) + } + } + + /// Fetch the [`info_json`], downloaded on-disk for this video. + /// + /// Will return [`None`], if the video does not have an downloaded [`info_json`] + pub(crate) fn get_info_json(&self) -> Result<Option<InfoJson>> { + if let Some(path) = self.info_json_path()? { + let info_json_string = File::open(path)?; + let info_json = serde_json::from_reader(&info_json_string)?; + + Ok(Some(info_json)) + } else { + Ok(None) + } + } + + /// Returns this videos `is_focused` flag if it is set. + /// + /// Will return `false` for not-downloaded videos. + pub(crate) fn is_focused(&self) -> bool { + if let VideoStatus::Cached { is_focused, .. } = &self.status { + *is_focused + } else { + false + } + } + + /// Returns the videos that are in the `allowed_states`. + /// + /// # Panics + /// Only, if assertions fail. + pub(crate) async fn in_states( + app: &App, + allowed_states: &[VideoStatusMarker], + ) -> Result<Vec<Video>> { + fn test(all_states: &[VideoStatusMarker], check: VideoStatusMarker) -> Option<i64> { + if all_states.contains(&check) { + Some(check.as_db_integer()) + } else { + None + } + } + fn states_to_string(allowed_states: &[VideoStatusMarker]) -> String { + let mut states = allowed_states + .iter() + .fold(String::from("&["), |mut acc, state| { + acc.push_str(state.as_str()); + acc.push_str(", "); + acc + }); + states = states.trim().to_owned(); + states = states.trim_end_matches(',').to_owned(); + states.push(']'); + states + } + + debug!( + "Fetching videos in the states: '{}'", + states_to_string(allowed_states) + ); + let active_pick: Option<i64> = test(allowed_states, VideoStatusMarker::Pick); + let active_watch: Option<i64> = test(allowed_states, VideoStatusMarker::Watch); + let active_cached: Option<i64> = test(allowed_states, VideoStatusMarker::Cached); + let active_watched: Option<i64> = test(allowed_states, VideoStatusMarker::Watched); + let active_drop: Option<i64> = test(allowed_states, VideoStatusMarker::Drop); + let active_dropped: Option<i64> = test(allowed_states, VideoStatusMarker::Dropped); + + // NOTE: The ORDER BY statement should be the same as the one in [`next_to_download`]. <2024-08-22> + let videos = query!( + r" + SELECT * + FROM videos + WHERE status IN (?,?,?,?,?,?) + ORDER BY priority DESC, publish_date DESC; + ", + active_pick, + active_watch, + active_cached, + active_watched, + active_drop, + active_dropped, + ) + .fetch_all(&app.database) + .await + .with_context(|| { + format!( + "Failed to query videos with states: '{}'", + states_to_string(allowed_states) + ) + })?; + + let real_videos: Vec<Video> = videos + .iter() + .map(|base| -> Video { video_from_record!(base) }) + .collect(); + + Ok(real_videos) + } +} diff --git a/crates/yt/src/storage/db/insert/maintenance.rs b/crates/yt/src/storage/db/insert/maintenance.rs new file mode 100644 index 0000000..d87c1ae --- /dev/null +++ b/crates/yt/src/storage/db/insert/maintenance.rs @@ -0,0 +1,38 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{ + insert::Operations, + video::{Video, VideoStatus, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +/// Remove the downloaded paths from videos in the db, that no longer exist on the file system. +pub(crate) async fn clear_stale_downloaded_paths(app: &App) -> Result<()> { + let mut cached_videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("DbMaintain: init"); + for vid in &mut cached_videos { + if let VideoStatus::Cached { cache_path, .. } = &vid.status { + if !cache_path.exists() { + vid.remove_download_path(&mut ops); + } + } else { + unreachable!("We only asked for cached videos.") + } + } + ops.commit(app).await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/db/insert/mod.rs b/crates/yt/src/storage/db/insert/mod.rs new file mode 100644 index 0000000..3458608 --- /dev/null +++ b/crates/yt/src/storage/db/insert/mod.rs @@ -0,0 +1,115 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::mem; + +use crate::app::App; + +use anyhow::Result; +use chrono::Utc; +use log::{debug, trace}; +use serde::{Serialize, de::DeserializeOwned}; +use sqlx::{SqliteConnection, query}; + +pub(crate) mod maintenance; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod video; + +pub(crate) trait Committable: + Sized + std::fmt::Debug + Serialize + DeserializeOwned +{ + async fn commit(self, txn: &mut SqliteConnection) -> Result<()>; +} + +#[derive(Debug)] +pub(crate) struct Operations<O: Committable> { + name: &'static str, + ops: Vec<O>, +} + +impl<O: Committable> Default for Operations<O> { + fn default() -> Self { + Self::new("<default impl>") + } +} + +impl<O: Committable> Operations<O> { + #[must_use] + pub(crate) fn new(name: &'static str) -> Self { + Self { + name, + ops: Vec::new(), + } + } + + pub(crate) async fn commit(mut self, app: &App) -> Result<()> { + let ops = mem::take(&mut self.ops); + + if ops.is_empty() { + return Ok(()); + } + + trace!("Begin commit of {}", self.name); + let mut txn = app.database.begin().await?; + + for op in ops { + trace!("Commiting operation: {op:?}"); + add_operation_to_txn_log(&op, &mut txn).await?; + op.commit(&mut txn).await?; + } + + txn.commit().await?; + trace!("End commit of {}", self.name); + + Ok(()) + } + + pub(crate) fn push(&mut self, op: O) { + self.ops.push(op); + } +} + +impl<O: Committable> Drop for Operations<O> { + fn drop(&mut self) { + assert!( + self.ops.is_empty(), + "Trying to drop uncommitted operations (name: {}) ({:#?}). This is a bug.", + self.name, + self.ops + ); + } +} + +async fn add_operation_to_txn_log<O: Committable>( + operation: &O, + txn: &mut SqliteConnection, +) -> Result<()> { + debug!("Adding operation to txn log: {operation:?}"); + + let now = Utc::now().timestamp(); + let operation = serde_json::to_string(&operation).expect("should be serializable"); + + query!( + r#" + INSERT INTO txn_log ( + timestamp, + operation + ) + VALUES (?, ?); + "#, + now, + operation, + ) + .execute(txn) + .await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/db/insert/playlist.rs b/crates/yt/src/storage/db/insert/playlist.rs new file mode 100644 index 0000000..4d3e140 --- /dev/null +++ b/crates/yt/src/storage/db/insert/playlist.rs @@ -0,0 +1,222 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{cmp::Ordering, time::Duration}; + +use anyhow::{Context, Result}; +use colors::Colorize; +use libmpv2::Mpv; +use log::{debug, trace}; + +use crate::{ + app::App, + storage::db::{ + insert::{Operations, video::Operation}, + playlist::{Playlist, PlaylistIndex}, + video::VideoStatus, + }, +}; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum VideoTransition { + Watched, + Picked, +} + +impl Playlist { + pub(crate) fn mark_current_done( + &mut self, + app: &App, + mpv: &Mpv, + new_state: VideoTransition, + ops: &mut Operations<Operation>, + ) -> Result<()> { + let (current_index, current_video) = self + .get_focused_mut() + .expect("This should be some at this point"); + + debug!( + "Playlist handler will mark video '{}' {:?}.", + current_video.title, new_state + ); + + match new_state { + VideoTransition::Watched => current_video.set_watched(ops), + VideoTransition::Picked => current_video.set_status(VideoStatus::Pick, ops), + } + + self.save_watch_progress(mpv, current_index, ops); + + self.videos.remove(Into::<usize>::into(current_index)); + + { + // Decide which video to mark focused now. + let index = usize::from(current_index); + let playlist_length = self.len(); + + if playlist_length == 0 { + // There are no new videos to mark focused. + } else { + let index = match index.cmp(&playlist_length) { + Ordering::Greater => { + unreachable!( + "The index '{index}' cannot exceed the \ + playlist length '{playlist_length}' as indices are 0 based." + ); + } + Ordering::Less => { + // The index is still valid. + // Therefore, we keep the user at this position. + index + } + Ordering::Equal => { + // The index is pointing to the end of the playlist. We could either go the second + // to last entry (i.e., one entry back) or wrap around to the start. + // We wrap around. + 0 + } + }; + + let next = self + .get_mut(PlaylistIndex::from(index)) + .expect("We checked that the index is still good"); + next.set_focused(true, ops); + } + + // Tell mpv about our decision. + self.resync_with_mpv(app, mpv)?; + } + + Ok(()) + } + + /// Sync the mpv playlist with this playlist. + pub(crate) fn resync_with_mpv(&self, app: &App, mpv: &Mpv) -> Result<()> { + fn get_playlist_count(mpv: &Mpv) -> Result<usize> { + mpv.get_property::<i64>("playlist/count") + .context("Failed to get mpv playlist len") + .map(|count| { + usize::try_from(count).expect("The playlist_count should always be positive") + }) + } + + if get_playlist_count(mpv)? != 0 { + // We could also use `loadlist`, but that would require use to start a unix socket or even + // write all the video paths to a file beforehand + mpv.command("playlist-clear", &[])?; + mpv.command("playlist-remove", &["current"])?; + } + + assert_eq!( + get_playlist_count(mpv)?, + 0, + "The playlist should be empty at this point." + ); + + debug!("MpvReload: Adding {} videos to playlist.", self.len()); + + self.videos + .iter() + .enumerate() + .try_for_each(|(index, video)| { + let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + else { + unreachable!("All of the videos in a playlist are cached"); + }; + + let options = format!( + "speed={},start={}", + video + .playback_speed + .unwrap_or(app.config.select.playback_speed), + i64::try_from(video.watch_progress.as_secs()) + .expect("This should not overflow"), + ); + + mpv.command( + "loadfile", + &[ + cache_path.to_str().with_context(|| { + format!( + "Failed to parse the video cache path ('{}') as valid utf8", + cache_path.display() + ) + })?, + "append-play", + "-1", // Not used for `append-play`, but needed for the next args to take effect. + options.as_str(), + ], + )?; + + if *is_focused { + debug!("MpvReload: Setting playlist position to {index}"); + mpv.set_property("playlist-pos", index.to_string().as_str())?; + } + + Ok::<(), anyhow::Error>(()) + })?; + + Ok(()) + } + + pub(crate) fn save_current_watch_progress( + &mut self, + mpv: &Mpv, + ops: &mut Operations<Operation>, + ) { + let (index, _) = self + .get_focused_mut() + .expect("This should be some at this point"); + + self.save_watch_progress(mpv, index, ops); + } + + /// Saves the `watch_progress` of a video at the index. + pub(crate) fn save_watch_progress( + &mut self, + mpv: &Mpv, + at: PlaylistIndex, + ops: &mut Operations<Operation>, + ) { + let current_video = self + .get_mut(at) + .expect("We should never produce invalid playlist indices"); + + let watch_progress = match mpv.get_property::<i64>("time-pos") { + Ok(time) => u64::try_from(time) + .expect("This conversion should never fail as the `time-pos` property is positive"), + Err(err) => { + // We cannot hard error here, as we would open us to an race condition between mpv + // changing the current video and we saving it. + trace!( + "While trying to save the watch progress for the current video: \ + Failed to get the watchprogress of the currently playling video: \ + (This is probably expected, nevertheless showing the raw error) \ + {err}" + ); + + return; + } + }; + + let watch_progress = Duration::from_secs(watch_progress); + + debug!( + "Setting the watch progress for the current_video '{}' to {}s", + current_video.title_fmt().render(false), + watch_progress.as_secs(), + ); + + current_video.set_watch_progress(watch_progress, ops); + } +} diff --git a/crates/yt/src/storage/db/insert/subscription.rs b/crates/yt/src/storage/db/insert/subscription.rs new file mode 100644 index 0000000..d25a209 --- /dev/null +++ b/crates/yt/src/storage/db/insert/subscription.rs @@ -0,0 +1,95 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::storage::db::{ + insert::{Committable, Operations}, + subscription::{Subscription, Subscriptions}, +}; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use sqlx::query; + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum Operation { + Add(Subscription), + Remove(Subscription), +} + +impl Committable for Operation { + async fn commit(self, txn: &mut sqlx::SqliteConnection) -> Result<()> { + match self { + Operation::Add(subscription) => { + let url = subscription.url.as_str(); + + query!( + " + INSERT INTO subscriptions ( + name, + url + ) VALUES (?, ?); + ", + subscription.name, + url + ) + .execute(txn) + .await?; + + println!( + "Subscribed to '{}' at '{}'", + subscription.name, subscription.url + ); + Ok(()) + } + Operation::Remove(subscription) => { + let output = query!( + " + DELETE FROM subscriptions + WHERE name = ? + ", + subscription.name, + ) + .execute(txn) + .await?; + + assert_eq!( + output.rows_affected(), + 1, + "The removed subscription query did effect more (or less) than one row. This is a bug." + ); + + println!( + "Unsubscribed from '{}' at '{}'", + subscription.name, subscription.url + ); + + Ok(()) + } + } + } +} + +impl Subscription { + pub(crate) fn add(self, ops: &mut Operations<Operation>) { + ops.push(Operation::Add(self)); + } + + pub(crate) fn remove(self, ops: &mut Operations<Operation>) { + ops.push(Operation::Remove(self)); + } +} + +impl Subscriptions { + pub(crate) fn remove(self, ops: &mut Operations<Operation>) { + for sub in self.0.into_values() { + ops.push(Operation::Remove(sub)); + } + } +} diff --git a/crates/yt/src/storage/db/insert/video/mod.rs b/crates/yt/src/storage/db/insert/video/mod.rs new file mode 100644 index 0000000..da62e37 --- /dev/null +++ b/crates/yt/src/storage/db/insert/video/mod.rs @@ -0,0 +1,610 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + path::{Path, PathBuf}, + time, +}; + +use crate::storage::db::{ + extractor_hash::ExtractorHash, + insert::{Committable, Operations}, + video::{Priority, Video, VideoStatus, VideoStatusMarker}, +}; + +use anyhow::{Context, Result}; +use chrono::Utc; +use log::debug; +use serde::{Deserialize, Serialize}; +use sqlx::query; +use tokio::fs; + +use super::super::video::TimeStamp; + +const fn is_focused_to_value(is_focused: bool) -> Option<i8> { + if is_focused { Some(1) } else { None } +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum Operation { + Add { + description: Option<String>, + title: String, + parent_subscription_name: Option<String>, + thumbnail_url: Option<String>, + url: String, + extractor_hash: String, + status: i64, + cache_path: Option<String>, + is_focused: Option<i8>, + duration: Option<f64>, + last_status_change: i64, + publish_date: Option<i64>, + watch_progress: i64, + }, + // TODO(@bpeetz): Could both the {`Set`,`Remove`}`DownloadPath` ops, be merged into SetStatus + // {`Cached`,`Watch`}? <2025-07-14> + SetDownloadPath { + video: ExtractorHash, + path: PathBuf, + }, + RemoveDownloadPath { + video: ExtractorHash, + }, + SetStatus { + video: ExtractorHash, + status: VideoStatus, + }, + SetPriority { + video: ExtractorHash, + priority: Priority, + }, + SetPlaybackSpeed { + video: ExtractorHash, + playback_speed: f64, + }, + SetSubtitleLangs { + video: ExtractorHash, + subtitle_langs: String, + }, + SetWatchProgress { + video: ExtractorHash, + watch_progress: time::Duration, + }, + SetIsFocused { + video: ExtractorHash, + is_focused: bool, + }, +} + +impl Committable for Operation { + #[allow(clippy::too_many_lines)] + async fn commit(self, txn: &mut sqlx::SqliteConnection) -> Result<()> { + match self { + Operation::SetDownloadPath { video, path } => { + debug!("Setting cache path from '{video}' to '{}'", path.display()); + + let path_str = path.display().to_string(); + let extractor_hash = video.hash().to_string(); + let status = VideoStatusMarker::Cached.as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = ?, status = ? + WHERE extractor_hash = ?; + "#, + path_str, + status, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::RemoveDownloadPath { video } => { + let extractor_hash = video.hash().to_string(); + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + let old = video.get(&mut *txn).await?; + + debug!("Deleting download path of '{video}' ({}).", old.title); + + if let VideoStatus::Cached { cache_path, .. } = &old.status { + if let Ok(true) = cache_path.try_exists() { + fs::remove_file(cache_path).await?; + } + + { + let info_json_path = old.info_json_path()?.expect("Is downloaded"); + + if let Ok(true) = info_json_path.try_exists() { + fs::remove_file(info_json_path).await?; + } + } + + { + if old.subtitle_langs.is_some() { + // TODO(@bpeetz): Also clean-up the downloaded subtitle files. <2025-07-05> + } + } + } else { + unreachable!( + "A video cannot have a download path deletion \ + queued without being marked as Cached." + ); + } + + query!( + r#" + UPDATE videos + SET cache_path = NULL, status = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + status, + None::<i32>, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetStatus { video, status } => { + let extractor_hash = video.hash().to_string(); + + let old = video.get(&mut *txn).await?; + + let old_marker = old.status.as_marker(); + + let (cache_path, is_focused) = { + fn cache_path_to_string(path: &Path) -> Result<String> { + Ok(path + .to_str() + .with_context(|| { + format!( + "Failed to parse cache path ('{}') as utf8 string", + path.display() + ) + })? + .to_owned()) + } + + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &status + { + ( + Some(cache_path_to_string(cache_path)?), + is_focused_to_value(*is_focused), + ) + } else { + (None, None) + } + }; + + let new_status = status.as_marker(); + + assert_ne!( + old_marker, new_status, + "We should have never generated this operation" + ); + + let now = Utc::now().timestamp(); + + debug!( + "Changing status of video ('{}' {extractor_hash}) \ + from {old_marker:#?} to {new_status:#?}.", + old.title + ); + + let new_status = new_status.as_db_integer(); + query!( + r#" + UPDATE videos + SET status = ?, last_status_change = ?, cache_path = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + new_status, + now, + cache_path, + is_focused, + extractor_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetPriority { video, priority } => { + let extractor_hash = video.hash().to_string(); + + let new_priority = priority.as_db_integer(); + query!( + r#" + UPDATE videos + SET priority = ? + WHERE extractor_hash = ?; + "#, + new_priority, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetWatchProgress { + video, + watch_progress, + } => { + let video_extractor_hash = video.hash().to_string(); + let watch_progress = i64::try_from(watch_progress.as_secs()) + .expect("This should never exceed its bounds"); + + query!( + r#" + UPDATE videos + SET watch_progress = ? + WHERE extractor_hash = ?; + "#, + watch_progress, + video_extractor_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetPlaybackSpeed { + video, + playback_speed, + } => { + let extractor_hash = video.hash().to_string(); + + query!( + r#" + UPDATE videos + SET playback_speed = ? + WHERE extractor_hash = ?; + "#, + playback_speed, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetSubtitleLangs { + video, + subtitle_langs, + } => { + let extractor_hash = video.hash().to_string(); + + query!( + r#" + UPDATE videos + SET subtitle_langs = ? + WHERE extractor_hash = ?; + "#, + subtitle_langs, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetIsFocused { video, is_focused } => { + debug!("Set is_focused of video: '{video}' to {is_focused}"); + let new_hash = video.hash().to_string(); + let new_is_focused = is_focused_to_value(is_focused); + + query!( + r#" + UPDATE videos + SET is_focused = ? + WHERE extractor_hash = ?; + "#, + new_is_focused, + new_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::Add { + parent_subscription_name, + thumbnail_url, + url, + extractor_hash, + status, + cache_path, + is_focused, + duration, + last_status_change, + publish_date, + watch_progress, + description, + title, + } => { + query!( + r#" + INSERT INTO videos ( + description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + title, + url, + watch_progress, + cache_path + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + "#, + description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + title, + url, + watch_progress, + cache_path, + ) + .execute(txn) + .await?; + + Ok(()) + } + } + } +} + +impl Video { + /// Add this in-memory video to the db. + pub(crate) fn add(self, ops: &mut Operations<Operation>) -> Result<Self> { + let description = self.description.clone(); + let title = self.title.clone(); + let parent_subscription_name = self.parent_subscription_name.clone(); + + let thumbnail_url = self.thumbnail_url.as_ref().map(ToString::to_string); + + let url = self.url.to_string(); + let extractor_hash = self.extractor_hash.hash().to_string(); + + let status = self.status.as_marker().as_db_integer(); + let (cache_path, is_focused) = if let VideoStatus::Cached { + cache_path, + is_focused, + } = &self.status + { + ( + Some( + cache_path + .to_str() + .with_context(|| { + format!( + "Failed to prase cache path '{}' as utf-8 string", + cache_path.display() + ) + })? + .to_string(), + ), + is_focused_to_value(*is_focused), + ) + } else { + (None, None) + }; + + let duration: Option<f64> = self.duration.as_secs_f64(); + let last_status_change: i64 = self.last_status_change.as_secs(); + let publish_date: Option<i64> = self.publish_date.map(TimeStamp::as_secs); + let watch_progress: i64 = + i64::try_from(self.watch_progress.as_secs()).expect("This should never exceed a u32"); + + ops.push(Operation::Add { + description, + title, + parent_subscription_name, + thumbnail_url, + url, + extractor_hash, + status, + cache_path, + is_focused, + duration, + last_status_change, + publish_date, + watch_progress, + }); + + Ok(self) + } + + /// Set the download path of a video. + /// + /// # Note + /// This will also set the status to `Cached`. + pub(crate) fn set_download_path(&mut self, path: &Path, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { cache_path, .. } = &mut self.status { + if cache_path != path { + // Update the in-memory video. + path.clone_into(cache_path); + + ops.push(Operation::SetDownloadPath { + video: self.extractor_hash, + path: path.to_owned(), + }); + } + } else { + self.status = VideoStatus::Cached { + cache_path: path.to_owned(), + is_focused: false, + }; + + ops.push(Operation::SetDownloadPath { + video: self.extractor_hash, + path: path.to_owned(), + }); + } + } + + /// Remove the download path of a video. + /// + /// # Note + /// This will also set the status to `Watch`. + /// + /// # Panics + /// If the status is not `Cached`. + pub(crate) fn remove_download_path(&mut self, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { .. } = &mut self.status { + self.status = VideoStatus::Watch; + ops.push(Operation::RemoveDownloadPath { + video: self.extractor_hash, + }); + } else { + unreachable!("Can only remove the path from a `Cached` video"); + } + } + + /// Update the `is_focused` flag of this video. + /// + /// # Note + /// It will only actually add operations, if the `is_focused` flag is different. + /// + /// # Panics + /// If the status is not `Cached`. + pub(crate) fn set_focused(&mut self, new_is_focused: bool, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { is_focused, .. } = &mut self.status { + if *is_focused != new_is_focused { + *is_focused = new_is_focused; + + ops.push(Operation::SetIsFocused { + video: self.extractor_hash, + is_focused: new_is_focused, + }); + } + } else { + unreachable!("Can only change `is_focused` on a Cached video."); + } + } + + /// Set the status of this video. + /// + /// # Note + /// This will not actually add any operations, if the new status equals the old one. + pub(crate) fn set_status(&mut self, status: VideoStatus, ops: &mut Operations<Operation>) { + if self.status != status { + status.clone_into(&mut self.status); + + ops.push(Operation::SetStatus { + video: self.extractor_hash, + status, + }); + } + } + + /// Set the priority of this video. + /// + /// # Note + /// This will not actually add any operations, if the new priority equals the old one. + pub(crate) fn set_priority(&mut self, priority: Priority, ops: &mut Operations<Operation>) { + if self.priority != priority { + self.priority = priority; + + ops.push(Operation::SetPriority { + video: self.extractor_hash, + priority, + }); + } + } + + /// Set the watch progress. + /// + /// # Note + /// This will not actually add any operations, + /// if the new watch progress equals the old one. + pub(crate) fn set_watch_progress( + &mut self, + watch_progress: time::Duration, + ops: &mut Operations<Operation>, + ) { + if self.watch_progress != watch_progress { + self.watch_progress = watch_progress; + + ops.push(Operation::SetWatchProgress { + video: self.extractor_hash, + watch_progress, + }); + } + } + + /// Set the playback speed of this video. + /// + /// # Note + /// This will not actually add any operations, if the new speed equals the old one. + pub(crate) fn set_playback_speed( + &mut self, + playback_speed: f64, + ops: &mut Operations<Operation>, + ) { + if self.playback_speed != Some(playback_speed) { + self.playback_speed = Some(playback_speed); + + ops.push(Operation::SetPlaybackSpeed { + video: self.extractor_hash, + playback_speed, + }); + } + } + + /// Set the subtitle langs of this video. + /// + /// # Note + /// This will not actually add any operations, if the new langs equal the old one. + pub(crate) fn set_subtitle_langs( + &mut self, + subtitle_langs: String, + ops: &mut Operations<Operation>, + ) { + if self.subtitle_langs.as_ref() != Some(&subtitle_langs) { + self.subtitle_langs = Some(subtitle_langs.clone()); + + ops.push(Operation::SetSubtitleLangs { + video: self.extractor_hash, + subtitle_langs, + }); + } + } + + /// Mark this video watched. + /// This will both set the status to `Watched` and the `cache_path` to Null. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) fn set_watched(&mut self, ops: &mut Operations<Operation>) { + self.remove_download_path(ops); + self.set_status(VideoStatus::Watched, ops); + } +} diff --git a/crates/yt/src/storage/db/mod.rs b/crates/yt/src/storage/db/mod.rs new file mode 100644 index 0000000..926bab0 --- /dev/null +++ b/crates/yt/src/storage/db/mod.rs @@ -0,0 +1,18 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod get; +pub(crate) mod insert; + +pub(crate) mod extractor_hash; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod txn_log; +pub(crate) mod video; diff --git a/crates/yt/src/storage/db/playlist/mod.rs b/crates/yt/src/storage/db/playlist/mod.rs new file mode 100644 index 0000000..7366e8e --- /dev/null +++ b/crates/yt/src/storage/db/playlist/mod.rs @@ -0,0 +1,59 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::ops::Add; + +use crate::storage::db::video::Video; + +/// Zero-based index into the internal playlist. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PlaylistIndex(usize); + +impl From<PlaylistIndex> for usize { + fn from(value: PlaylistIndex) -> Self { + value.0 + } +} + +impl From<usize> for PlaylistIndex { + fn from(value: usize) -> Self { + Self(value) + } +} + +impl Add<usize> for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl Add for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +/// A representation of the internal Playlist +#[derive(Debug)] +pub(crate) struct Playlist { + pub(crate) videos: Vec<Video>, +} + +impl Playlist { + /// Returns the number of videos in the playlist + #[must_use] + pub(crate) fn len(&self) -> usize { + self.videos.len() + } +} diff --git a/crates/yt/src/storage/db/subscription.rs b/crates/yt/src/storage/db/subscription.rs new file mode 100644 index 0000000..c111b52 --- /dev/null +++ b/crates/yt/src/storage/db/subscription.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::collections::HashMap; + +use anyhow::Result; +use log::debug; +use serde::{Deserialize, Serialize}; +use url::Url; +use yt_dlp::{json_cast, json_try_get, options::YoutubeDLOptions}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct Subscription { + /// The human readable name of this subscription + pub(crate) name: String, + + /// The URL this subscription subscribes to + pub(crate) url: Url, +} + +impl Subscription { + #[must_use] + pub(crate) fn new(name: String, url: Url) -> Self { + Self { name, url } + } +} + +#[derive(Default, Debug)] +pub(crate) struct Subscriptions(pub(crate) HashMap<String, Subscription>); + +/// Check whether an URL could be used as a subscription URL +pub(crate) async fn check_url(url: Url) -> Result<bool> { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + debug!("{info:#?}"); + + Ok(json_try_get!(info, "_type", as_str) == Some("playlist")) +} diff --git a/crates/yt/src/storage/db/txn_log.rs b/crates/yt/src/storage/db/txn_log.rs new file mode 100644 index 0000000..64884b0 --- /dev/null +++ b/crates/yt/src/storage/db/txn_log.rs @@ -0,0 +1,24 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::storage::db::{insert::Committable, video::TimeStamp}; + +pub(crate) struct TxnLog<O: Committable> { + inner: Vec<(TimeStamp, O)>, +} + +impl<O: Committable> TxnLog<O> { + pub(crate) fn new(inner: Vec<(TimeStamp, O)>) -> Self { + Self { inner } + } + pub(crate) fn inner(&self) -> &[(TimeStamp, O)] { + &self.inner + } +} diff --git a/yt/src/comments/display.rs b/crates/yt/src/storage/db/video/comments/display.rs index 6166b2b..c372603 100644 --- a/yt/src/comments/display.rs +++ b/crates/yt/src/storage/db/video/comments/display.rs @@ -1,6 +1,5 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> // Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // @@ -13,27 +12,22 @@ use std::fmt::Write; use chrono::{Local, TimeZone}; use chrono_humanize::{Accuracy, HumanTime, Tense}; +use colors::{Colorize, IntoCanvas}; -use crate::comments::comment::CommentExt; - -use super::comment::Comments; +use crate::{ + output::format_text, + storage::db::video::comments::{Comment, Comments}, +}; impl Comments { - pub fn render(&self, color: bool) -> String { - self.render_help(color).expect("This should never fail.") + pub(crate) fn render(&self, use_color: bool) -> String { + self.render_help(use_color) + .expect("This should never fail.") } - fn render_help(&self, color: bool) -> Result<String, std::fmt::Error> { - macro_rules! c { - ($color_str:expr, $write:ident, $color:expr) => { - if $color { - $write.write_str(concat!("\x1b[", $color_str, "m"))? - } - }; - } - + fn render_help(&self, use_color: bool) -> Result<String, std::fmt::Error> { fn format( - comment: &CommentExt, + comment: &Comment, f: &mut String, ident_count: u32, color: bool, @@ -43,14 +37,16 @@ impl Comments { f.write_str(ident)?; - if value.author_is_uploader { - c!("91;1", f, color); - } else { - c!("35", f, color); - } + write!( + f, + "{}", + if value.author_is_uploader { + (&value.author).bold().bright_red().render(color) + } else { + (&value.author).purple().render(color) + } + )?; - f.write_str(&value.author)?; - c!("0", f, color); if value.edited || value.is_favorited { f.write_str("[")?; if value.edited { @@ -65,7 +61,6 @@ impl Comments { f.write_str("]")?; } - c!("36;1", f, color); write!( f, " {}", @@ -76,17 +71,31 @@ impl Comments { .expect("This should be valid") ) .to_text_en(Accuracy::Rough, Tense::Past) + .bold() + .cyan() + .render(color) )?; - c!("0", f, color); - // c!("31;1", f); - // f.write_fmt(format_args!(" [{}]", comment.value.like_count))?; - // c!("0", f); + write!( + f, + " [{}]", + comment.value.like_count.bold().red().render(color) + )?; f.write_str(":\n")?; f.write_str(ident)?; - f.write_str(&value.text.replace('\n', &format!("\n{ident}")))?; + f.write_str( + &format_text( + value.text.trim(), + Some( + termsize::get().map_or(90, |ts| ts.cols) + - u16::try_from(ident_count).expect("Should never overflow"), + ), + ) + .trim() + .replace('\n', &format!("\n{ident}")), + )?; f.write_str("\n")?; if comment.replies.is_empty() { @@ -105,12 +114,12 @@ impl Comments { let mut f = String::new(); - if !&self.vec.is_empty() { - let mut children = self.vec.clone(); + if !&self.inner.is_empty() { + let mut children = self.inner.clone(); children.sort_by(|a, b| b.value.like_count.cmp(&a.value.like_count)); for child in children { - format(&child, &mut f, 0, color)?; + format(&child, &mut f, 0, use_color)?; } } Ok(f) diff --git a/crates/yt/src/storage/db/video/comments/mod.rs b/crates/yt/src/storage/db/video/comments/mod.rs new file mode 100644 index 0000000..41a03be --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/mod.rs @@ -0,0 +1,202 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::mem; + +use regex::{Captures, Regex}; + +use crate::storage::db::video::comments::raw::{Parent, RawComment}; + +pub(crate) mod display; +pub(crate) mod raw; + +#[cfg(test)] +mod tests; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Comment { + value: RawComment, + replies: Vec<Self>, +} + +#[derive(Debug, Default, PartialEq)] +pub(crate) struct Comments { + inner: Vec<Comment>, +} + +impl Comments { + pub(crate) fn from_raw(raw: Vec<RawComment>) -> Self { + let mut me = Self::default(); + + // Apply the parent -> child mapping yt provides us with. + for raw_comment in raw { + if let Parent::Id(id) = &raw_comment.parent { + me.insert(&(id.clone()), Comment::from(raw_comment)); + } else { + me.inner.push(Comment::from(raw_comment)); + } + } + + { + // Sort the final comments chronologically. + // This ensures that replies are matched with the comment they actually replied to and + // not a later comment from the same author. + for comment in &mut me.inner { + comment + .replies + .sort_by_key(|comment| comment.value.timestamp); + + for reply in &comment.replies { + assert!(reply.replies.is_empty()); + } + } + } + + { + let find_reply_indicator = + Regex::new(r"\u{200b}?(@[^\t\s]+)\u{200b}?").expect("This is hardcoded"); + + // Try to re-construct the replies for the reply comments. + for comment in &mut me.inner { + let previous_replies = mem::take(&mut comment.replies); + + let mut reply_tree = Comments::default(); + + for reply in previous_replies { + // We try to reconstruct the parent child relation ship by looking (naively) + // for a reply indicator. Currently, this is just the `@<some_name>`, as yt + // seems to insert that by default if you press `reply-to` in their clients. + // + // This follows these steps: + // - Does this reply have a “reply indicator”? + // - If yes, try to resolve the indicator. + // - If it is resolvable, add this reply to the [`Comment`] it resolved to. + // - If not, keep the comment as reply. + + if let Some(reply_indicator_matches) = + find_reply_indicator.captures(&reply.value.text.clone()) + { + // We found a reply indicator. + // First we traverse the current `reply_tree` in reversed order to find a + // match, than we check if the reply indicator matches the reply tree root + // and afterward we declare it unmatching and add it as toplevel. + + let reply_target_author = reply_indicator_matches + .get(1) + .expect("This should also exist") + .as_str(); + + if let Some(parent) = reply_tree.find_author_mut(reply_target_author) { + parent + .replies + .push(comment_from_reply(reply, &reply_indicator_matches)); + } else if comment.value.author == reply_target_author { + reply_tree + .add_toplevel(comment_from_reply(reply, &reply_indicator_matches)); + } else { + eprintln!( + "Failed to find a parent for ('{}') both directly \ + and via replies! The reply text was:\n'{}'\n", + reply_target_author, reply.value.text + ); + reply_tree.add_toplevel(reply); + } + } else { + // The comment text did not contain a reply indicator, so add it as + // toplevel. + reply_tree.add_toplevel(reply); + } + } + + comment.replies = reply_tree.inner; + } + } + + me + } + + fn add_toplevel(&mut self, value: Comment) { + self.inner.push(value); + } + + fn insert(&mut self, id: &str, value: Comment) { + let parent = self + .inner + .iter_mut() + .find(|c| c.value.id.id == id) + .expect("One of these should exist"); + + parent.replies.push(value); + } + + fn find_author_mut(&mut self, reply_target_author: &str) -> Option<&mut Comment> { + fn perform_check<'a>( + comment: &'a mut Comment, + reply_target_author: &str, + ) -> Option<&'a mut Comment> { + // TODO(@bpeetz): This is a workaround until rust has lexiographic lifetime support. <2025-07-18> + fn find_in_replies<'a>( + comment: &'a mut Comment, + reply_target_author: &str, + ) -> Option<&'a mut Comment> { + comment + .replies + .iter_mut() + .rev() + .find_map(|reply: &mut Comment| perform_check(reply, reply_target_author)) + } + let comment_author_matches_target = comment.value.author == reply_target_author; + + match find_in_replies(comment, reply_target_author) { + Some(_) => Some( + // PERFORMANCE(@bpeetz): We should not need to run this code twice. <2025-07-18> + find_in_replies(comment, reply_target_author) + .expect("We already had a Some result for this."), + ), + None if comment_author_matches_target => Some(comment), + None => None, + } + } + + for comment in self.inner.iter_mut().rev() { + if let Some(output) = perform_check(comment, reply_target_author) { + return Some(output); + } + } + + None + } +} +fn comment_from_reply(reply: Comment, reply_indicator_matches: &Captures<'_>) -> Comment { + Comment::from(RawComment { + text: { + // Remove the `@<some_name>` for the comment text. + let full_match = reply_indicator_matches + .get(0) + .expect("This will always exist"); + + let text = reply.value.text[0..full_match.start()].to_owned() + + &reply.value.text[full_match.end()..]; + + text.trim_matches(|c: char| c == '\u{200b}' || c == '\u{2060}' || c.is_whitespace()) + .to_owned() + }, + ..reply.value + }) +} + +impl From<RawComment> for Comment { + fn from(value: RawComment) -> Self { + Self { + value, + replies: vec![], + } + } +} diff --git a/crates/yt/src/storage/db/video/comments/raw.rs b/crates/yt/src/storage/db/video/comments/raw.rs new file mode 100644 index 0000000..3b7f40f --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/raw.rs @@ -0,0 +1,87 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use serde::{Deserialize, Deserializer}; +use url::Url; + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub(crate) struct Id { + pub(crate) id: String, +} +impl From<String> for Id { + fn from(value: String) -> Self { + Self { + // Take the last element if the string is split with dots, otherwise take the full id + id: value.split('.').next_back().unwrap_or(&value).to_owned(), + } + } +} + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub(crate) enum Parent { + Root, + Id(String), +} + +impl From<String> for Parent { + fn from(value: String) -> Self { + if value == "root" { + Self::Root + } else { + Self::Id(value) + } + } +} + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[allow(clippy::struct_excessive_bools)] +pub(crate) struct RawComment { + pub(crate) id: Id, + pub(crate) text: String, + #[serde(default = "zero")] + pub(crate) like_count: u32, + pub(crate) is_pinned: bool, + pub(crate) author_id: String, + #[serde(default = "unknown")] + pub(crate) author: String, + pub(crate) author_is_verified: bool, + pub(crate) author_thumbnail: Url, + pub(crate) parent: Parent, + #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] + pub(crate) edited: bool, + // Can't also be deserialized, as it's already used in 'edited' + // _time_text: String, + pub(crate) timestamp: i64, + pub(crate) author_url: Option<Url>, + pub(crate) author_is_uploader: bool, + pub(crate) is_favorited: bool, +} + +fn unknown() -> String { + "<Unknown>".to_string() +} +fn zero() -> u32 { + 0 +} +fn edited_from_time_text<'de, D>(d: D) -> Result<bool, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(d)?; + if s.contains(" (edited)") { + Ok(true) + } else { + Ok(false) + } +} diff --git a/crates/yt/src/storage/db/video/comments/tests.rs b/crates/yt/src/storage/db/video/comments/tests.rs new file mode 100644 index 0000000..03e3597 --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/tests.rs @@ -0,0 +1,249 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use pretty_assertions::assert_eq; +use url::Url; + +use crate::storage::db::video::comments::{ + Comment, Comments, RawComment, + raw::{Id, Parent}, +}; + +/// Generate both an [`expected`] and an [`input`] value from an expected comment expression. +macro_rules! mk_comments { + () => {{ + let input: Vec<RawComment> = vec![]; + let expected: Comments = Comments { + inner: vec![], + }; + + (input, expected) + }}; + + ( + $( + parent: $parent:expr, $actual_parent:ident, + ( + @ $name:ident : $comment:literal + $( + $reply_chain:tt + )* + ) + )+ + ) => {{ + let (nested_input, _) = mk_comments!( + $( + $( + parent: $parent, $name, + $reply_chain + )* + )+ + ); + + let mut input: Vec<RawComment> = vec![ + $( + mk_comments!(@to_raw input $name $comment $parent, $actual_parent) + ),+ + ]; + input.extend(nested_input); + + let expected: Comments = Comments { + inner: vec![ + $( + Comment { + value: mk_comments!(@to_raw expected $name $comment $parent, $actual_parent), + replies: { + let (_, nested_expected) = mk_comments!( + $( + parent: $parent, $name, + $reply_chain + )* + ); + + nested_expected.inner + }, + } + ),+ + ] + }; + + (input, expected) + }}; + ( + $( + ( + @ $name:ident : $comment:literal + $( + $reply_chain:tt + )* + ) + )+ + ) => {{ + let (nested_input, _) = mk_comments!( + $( + $( + parent: mk_comments!(@mk_id $name $comment), $name, + $reply_chain + )* + )+ + ); + + let mut input: Vec<RawComment> = vec![ + $( + mk_comments!(@to_raw input $name $comment) + ),+ + ]; + input.extend(nested_input); + + let expected: Comments = Comments { + inner: vec![ + $( + Comment { + value: mk_comments!(@to_raw expected $name $comment), + replies: { + let (_, nested_expected) = mk_comments!( + $( + parent: mk_comments!(@mk_id $name $comment), $name, + $reply_chain + )* + ); + + nested_expected.inner + }, + } + ),+ + ] + }; + + (input, expected) + }}; + + (@mk_id $name:ident $comment:literal) => {{ + use std::hash::{Hash, Hasher}; + + let input = format!("{}{}", stringify!($name), $comment); + + let mut digest = std::hash::DefaultHasher::new(); + input.hash(&mut digest); + Id { id: digest.finish().to_string() } + }}; + + (@to_raw $state:ident $name:ident $comment:literal $($parent:expr, $actual_parent:ident)?) => { + RawComment { + id: mk_comments!(@mk_id $name $comment), + text: mk_comments!(@mk_text $state $comment $(, $actual_parent)?), + like_count: 0, + is_pinned: false, + author_id: stringify!($name).to_owned(), + author: format!("@{}", stringify!($name)), + author_is_verified: false, + author_thumbnail: Url::from_file_path("/dev/null").unwrap(), + parent: mk_comments!(@mk_parent $($parent)?), + edited: false, + timestamp: 0, + author_url: None, + author_is_uploader: false, + is_favorited: false, + } + }; + + (@mk_parent) => { + Parent::Root + }; + (@mk_parent $parent:expr) => { + Parent::Id($parent.id) + }; + + (@mk_text input $text:expr) => { + $text.to_owned() + }; + (@mk_text input $text:expr, $actual_parent:ident) => { + format!("@{} {}", stringify!($actual_parent), $text) + }; + (@mk_text expected $text:expr $(, $_:tt)?) => { + $text.to_owned() + }; +} + +#[test] +fn test_comments_toplevel() { + let (input, expected) = mk_comments!( + (@kant: "I think, that using the results of an action to determine morality is flawed.") + (@hume: "I think, that we should use our feeling for morality more.") + (@lock: "I think, that we should rely on the sum of happiness caused by an action to determine it's morality.") + ); + + assert_eq!(Comments::from_raw(input), expected); +} + +#[test] +fn test_comments_replies_1_level() { + let (input, expected) = mk_comments!( + (@hume: "I think, that we should use our feeling for morality more." + (@kant: "This is so wrong! I shall now dedicate my next 7? years to writing books that prove this.") + (@lock: "It feels not very applicable, no? We should focus on something that can be used in the court of law!")) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_replies_2_levels() { + let (input, expected) = mk_comments!( + (@singer: "We perform medical studies on animals; Children have lower or similar mental ability as these animals.." + (@singer: "Therefore, we should perform these studies on children instead, if we were to follow our own principals" + (@james: "This is ridiculous! I will not entertain this thought.") + (@singer: "Although one could also use this argument to argue for abortion _after_ birth."))) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_replies_3_levels() { + let (input, expected) = mk_comments!( + (@singer: "We perform medical studies on animals; Children have lower or similar mental ability as these animals.." + (@singer: "Therefore, we should perform these studies on children instead, if we were to follow our own principals" + (@james: "This is ridiculous! I will not entertain this thought." + (@singer: "You know that I am not actually suggesting that? This is but a way to critizise the society")) + (@singer: "Although one could also use this argument to argue for abortion _after_ birth."))) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_sub_answer_selection() { + let (input, expected) = mk_comments!( + (@coffeewolfproductions9113: "I mean, brothels and sex workers in of themselves are not a bad thing." + (@aikikaname6508: "probably not so much in the 50s, pre contraception") + (@as_ri1mb: "it’s an incredibly sad, degrading line of work, often resulting in self loathing and self-deletion." + (@coffeewolfproductions9113: "Are you speaking from experience?" + (@as_ri1mb: "what an immature response, as expected." + (@coffeewolfproductions9113: "I literally just asked if you were talking from experience."))))) + + ); + + eprintln!("{}", expected.render(true)); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} diff --git a/yt/src/storage/video_database/mod.rs b/crates/yt/src/storage/db/video/mod.rs index 74d09f0..deeb82c 100644 --- a/yt/src/storage/video_database/mod.rs +++ b/crates/yt/src/storage/db/video/mod.rs @@ -1,6 +1,5 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> // Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // @@ -9,55 +8,108 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use std::{ - fmt::{Display, Write}, - path::PathBuf, - time::Duration, -}; +use std::{fmt::Display, path::PathBuf, time::Duration}; use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; use url::Url; -use crate::{ - app::App, select::selection_file::duration::MaybeDuration, - storage::video_database::extractor_hash::ExtractorHash, -}; +use crate::{select::duration::MaybeDuration, storage::db::extractor_hash::ExtractorHash}; -pub mod downloader; -pub mod extractor_hash; -pub mod get; -pub mod notify; -pub mod set; +pub(crate) mod comments; + +macro_rules! video_from_record { + ($record:expr) => { + $crate::storage::db::video::Video { + description: $record.description.clone(), + duration: $crate::select::duration::MaybeDuration::from_maybe_secs_f64( + $record.duration, + ), + extractor_hash: $crate::storage::db::extractor_hash::ExtractorHash::from_hash( + $record + .extractor_hash + .parse() + .expect("The db hash should be a valid blake3 hash"), + ), + last_status_change: $crate::storage::db::video::TimeStamp::from_secs( + $record.last_status_change, + ), + parent_subscription_name: $record.parent_subscription_name.clone(), + publish_date: $record + .publish_date + .map(|pd| $crate::storage::db::video::TimeStamp::from_secs(pd)), + status: { + let marker = + $crate::storage::db::video::VideoStatusMarker::from_db_integer($record.status); + let optional = if let Some(cache_path) = &$record.cache_path { + Some(( + std::path::PathBuf::from(cache_path), + if $record.is_focused == Some(1) { + true + } else { + false + }, + )) + } else { + None + }; + $crate::storage::db::video::VideoStatus::from_marker(marker, optional) + }, + thumbnail_url: if let Some(url) = &$record.thumbnail_url { + Some(url::Url::parse(url).expect("Parsing this as url should always work")) + } else { + None + }, + title: $record.title.clone(), + url: url::Url::parse(&$record.url).expect("Parsing this as url should always work"), + priority: $crate::storage::db::video::Priority::from($record.priority), + watch_progress: std::time::Duration::from_secs( + u64::try_from($record.watch_progress).expect("The record is positive i64"), + ), + subtitle_langs: $record.subtitle_langs.clone(), + playback_speed: $record.playback_speed, + } + }; +} +pub(crate) use video_from_record; #[derive(Debug, Clone)] -pub struct Video { - pub description: Option<String>, - pub duration: MaybeDuration, - pub extractor_hash: ExtractorHash, - pub last_status_change: TimeStamp, +pub(crate) struct Video { + pub(crate) description: Option<String>, + pub(crate) duration: MaybeDuration, + pub(crate) extractor_hash: ExtractorHash, + pub(crate) last_status_change: TimeStamp, /// The associated subscription this video was fetched from (null, when the video was `add`ed) - pub parent_subscription_name: Option<String>, - pub priority: Priority, - pub publish_date: Option<TimeStamp>, - pub status: VideoStatus, - pub thumbnail_url: Option<Url>, - pub title: String, - pub url: Url, + pub(crate) parent_subscription_name: Option<String>, + pub(crate) priority: Priority, + pub(crate) publish_date: Option<TimeStamp>, + pub(crate) status: VideoStatus, + pub(crate) thumbnail_url: Option<Url>, + pub(crate) title: String, + pub(crate) url: Url, /// The seconds the user has already watched the video - pub watch_progress: Duration, + pub(crate) watch_progress: Duration, + + /// Which subtitles to include, when downloading this video. + /// In the form of `lang1,lang2,lang3` (e.g. `en,de,sv`) + pub(crate) subtitle_langs: Option<String>, + + /// The playback speed to use, when watching this video. + /// Value is in percent, so 1 is 100%, 2.7 is 270%, and so on. + pub(crate) playback_speed: Option<f64>, } /// The priority of a [`Video`]. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct Priority { +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct Priority { value: i64, } impl Priority { /// Return the underlying value to insert that into the database #[must_use] - pub fn as_db_integer(&self) -> i64 { + pub(crate) fn as_db_integer(self) -> i64 { self.value } } @@ -74,25 +126,25 @@ impl Display for Priority { /// An UNIX time stamp. #[derive(Debug, Default, Clone, Copy)] -pub struct TimeStamp { +pub(crate) struct TimeStamp { value: i64, } impl TimeStamp { /// Return the seconds since the UNIX epoch for this [`TimeStamp`]. #[must_use] - pub fn as_secs(&self) -> i64 { + pub(crate) fn as_secs(self) -> i64 { self.value } /// Construct a [`TimeStamp`] from a count of seconds since the UNIX epoch. #[must_use] - pub fn from_secs(value: i64) -> Self { + pub(crate) fn from_secs(value: i64) -> Self { Self { value } } /// Construct a [`TimeStamp`] from the current time. #[must_use] - pub fn from_now() -> Self { + pub(crate) fn from_now() -> Self { Self { value: Utc::now().timestamp(), } @@ -107,49 +159,6 @@ impl Display for TimeStamp { } } -#[derive(Debug)] -pub struct VideoOptions { - pub yt_dlp: YtDlpOptions, - pub mpv: MpvOptions, -} -impl VideoOptions { - pub(crate) fn new(subtitle_langs: String, playback_speed: f64) -> Self { - let yt_dlp = YtDlpOptions { subtitle_langs }; - let mpv = MpvOptions { playback_speed }; - Self { yt_dlp, mpv } - } - - /// This will write out the options that are different from the defaults. - /// Beware, that this does not set the priority. - #[must_use] - pub fn to_cli_flags(self, app: &App) -> String { - let mut f = String::new(); - - if (self.mpv.playback_speed - app.config.select.playback_speed).abs() > f64::EPSILON { - write!(f, " --speed '{}'", self.mpv.playback_speed).expect("Works"); - } - if self.yt_dlp.subtitle_langs != app.config.select.subtitle_langs { - write!(f, " --subtitle-langs '{}'", self.yt_dlp.subtitle_langs).expect("Works"); - } - - f.trim().to_owned() - } -} - -#[derive(Debug, Clone, Copy)] -/// Additionally settings passed to mpv on watch -pub struct MpvOptions { - /// The playback speed. (1 is 100%, 2.7 is 270%, and so on) - pub playback_speed: f64, -} - -#[derive(Debug)] -/// Additionally configuration options, passed to yt-dlp on download -pub struct YtDlpOptions { - /// In the form of `lang1,lang2,lang3` (e.g. `en,de,sv`) - pub subtitle_langs: String, -} - /// # Video Lifetime (words in <brackets> are commands): /// <Pick> /// / \ @@ -158,8 +167,8 @@ pub struct YtDlpOptions { /// Cache // yt cache /// | /// Watched // yt watch -#[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] -pub enum VideoStatus { +#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub(crate) enum VideoStatus { #[default] Pick, @@ -186,7 +195,10 @@ impl VideoStatus { /// # Panics /// Only if internal expectations fail. #[must_use] - pub fn from_marker(marker: VideoStatusMarker, optional: Option<(PathBuf, bool)>) -> Self { + pub(crate) fn from_marker( + marker: VideoStatusMarker, + optional: Option<(PathBuf, bool)>, + ) -> Self { match marker { VideoStatusMarker::Pick => Self::Pick, VideoStatusMarker::Watch => Self::Watch, @@ -204,26 +216,9 @@ impl VideoStatus { } } - /// Turn the [`VideoStatus`] to its internal parts. This is only really useful for the database - /// functions. - #[must_use] - pub fn to_parts_for_db(self) -> (VideoStatusMarker, Option<(PathBuf, bool)>) { - match self { - VideoStatus::Pick => (VideoStatusMarker::Pick, None), - VideoStatus::Watch => (VideoStatusMarker::Watch, None), - VideoStatus::Cached { - cache_path, - is_focused, - } => (VideoStatusMarker::Cached, Some((cache_path, is_focused))), - VideoStatus::Watched => (VideoStatusMarker::Watched, None), - VideoStatus::Drop => (VideoStatusMarker::Drop, None), - VideoStatus::Dropped => (VideoStatusMarker::Dropped, None), - } - } - /// Return the associated [`VideoStatusMarker`] for this [`VideoStatus`]. #[must_use] - pub fn as_marker(&self) -> VideoStatusMarker { + pub(crate) fn as_marker(&self) -> VideoStatusMarker { match self { VideoStatus::Pick => VideoStatusMarker::Pick, VideoStatus::Watch => VideoStatusMarker::Watch, @@ -237,7 +232,7 @@ impl VideoStatus { /// Unit only variant of [`VideoStatus`] #[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] -pub enum VideoStatusMarker { +pub(crate) enum VideoStatusMarker { #[default] Pick, @@ -255,7 +250,7 @@ pub enum VideoStatusMarker { } impl VideoStatusMarker { - pub const ALL: &'static [Self; 6] = &[ + pub(crate) const ALL: &'static [Self; 6] = &[ Self::Pick, // Self::Watch, @@ -267,7 +262,7 @@ impl VideoStatusMarker { ]; #[must_use] - pub fn as_command(&self) -> &str { + pub(crate) fn as_command(&self) -> &str { // NOTE: Keep the serialize able variants synced with the main `select` function <2024-06-14> // Also try to ensure, that the strings have the same length match self { @@ -281,7 +276,7 @@ impl VideoStatusMarker { } #[must_use] - pub fn as_db_integer(&self) -> i64 { + pub(crate) fn as_db_integer(self) -> i64 { // These numbers should not change their mapping! // Oh, and keep them in sync with the SQLite check constraint. match self { @@ -296,7 +291,7 @@ impl VideoStatusMarker { } } #[must_use] - pub fn from_db_integer(num: i64) -> Self { + pub(crate) fn from_db_integer(num: i64) -> Self { match num { 0 => Self::Pick, @@ -314,7 +309,7 @@ impl VideoStatusMarker { } #[must_use] - pub fn as_str(&self) -> &'static str { + pub(crate) fn as_str(self) -> &'static str { match self { Self::Pick => "Pick", diff --git a/yt/src/storage/migrate/mod.rs b/crates/yt/src/storage/migrate/mod.rs index badeb6f..418c893 100644 --- a/yt/src/storage/migrate/mod.rs +++ b/crates/yt/src/storage/migrate/mod.rs @@ -21,8 +21,61 @@ use sqlx::{Sqlite, SqlitePool, Transaction, query}; use crate::app::App; +macro_rules! make_upgrade { + ($app:expr, $old_version:expr, $new_version:expr, $sql_name:expr) => { + add_error_context( + async { + let mut tx = $app + .database + .begin() + .await + .context("Failed to start the update transaction")?; + debug!("Migrating: {} -> {}", $old_version, $new_version); + + sqlx::raw_sql(include_str!($sql_name)) + .execute(&mut *tx) + .await + .context("Failed to run the update sql script")?; + + set_db_version( + &mut tx, + if $old_version == Self::Empty { + // There is no previous version we would need to remove + None + } else { + Some($old_version) + }, + $new_version, + ) + .await + .with_context(|| format!("Failed to set the new version ({})", $new_version))?; + + tx.commit() + .await + .context("Failed to commit the update transaction")?; + + // NOTE: This is needed, so that sqlite "sees" our changes to the table + // without having to reconnect. <2025-02-18> + query!("VACUUM") + .execute(&$app.database) + .await + .context("Failed to vacuum database")?; + + Ok(()) + }, + $new_version, + ) + .await?; + + Box::pin($new_version.update($app)).await.context(concat!( + "While updating to version: ", + stringify!($new_version) + )) + }; +} + #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] -pub enum DbVersion { +pub(crate) enum DbVersion { /// The database is not yet initialized. Empty, @@ -35,8 +88,17 @@ pub enum DbVersion { /// Introduced: 2025-02-18. Two, + + /// Introduced: 2025-03-21. + Three, + + /// Introduced: 2025-07-05. + Four, + + /// Introduced: 2025-07-20. + Five, } -const CURRENT_VERSION: DbVersion = DbVersion::Two; +const CURRENT_VERSION: DbVersion = DbVersion::Five; async fn add_error_context( function: impl Future<Output = Result<()>>, @@ -44,7 +106,7 @@ async fn add_error_context( ) -> Result<()> { function .await - .with_context(|| format!("Format failed to migrate database to version: {level}")) + .with_context(|| format!("Failed to migrate database to version: {level}")) } async fn set_db_version( @@ -83,21 +145,32 @@ async fn set_db_version( impl DbVersion { fn as_sql_integer(self) -> i32 { match self { - DbVersion::Empty => unreachable!("A empty version does not have an associated integer"), DbVersion::Zero => 0, DbVersion::One => 1, DbVersion::Two => 2, + DbVersion::Three => 3, + DbVersion::Four => 4, + DbVersion::Five => 5, + + DbVersion::Empty => unreachable!("A empty version does not have an associated integer"), } } + fn from_db(number: i64, namespace: &str) -> Result<Self> { match (number, namespace) { (0, "yt") => Ok(DbVersion::Zero), (1, "yt") => Ok(DbVersion::One), (2, "yt") => Ok(DbVersion::Two), + (3, "yt") => Ok(DbVersion::Three), + (4, "yt") => Ok(DbVersion::Four), + (5, "yt") => Ok(DbVersion::Five), (0, other) => bail!("Db version is Zero, but got unknown namespace: '{other}'"), (1, other) => bail!("Db version is One, but got unknown namespace: '{other}'"), (2, other) => bail!("Db version is Two, but got unknown namespace: '{other}'"), + (3, other) => bail!("Db version is Three, but got unknown namespace: '{other}'"), + (4, other) => bail!("Db version is Four, but got unknown namespace: '{other}'"), + (5, other) => bail!("Db version is Five, but got unknown namespace: '{other}'"), (other, "yt") => bail!("Got unkown version for 'yt' namespace: {other}"), (num, nasp) => bail!("Got unkown version number ({num}) and namespace ('{nasp}')"), @@ -111,126 +184,32 @@ impl DbVersion { #[allow(clippy::too_many_lines)] async fn update(self, app: &App) -> Result<()> { match self { - DbVersion::Empty => { - add_error_context( - async { - let mut tx = app - .database - .begin() - .await - .context("Failed to start transaction")?; - debug!("Migrate: Empty -> Zero"); - - sqlx::raw_sql(include_str!("./sql/00_empty_to_zero.sql")) - .execute(&mut *tx) - .await - .context("Failed to execute sql update script")?; - - set_db_version(&mut tx, None, DbVersion::Zero) - .await - .context("Failed to set new version")?; - - tx.commit() - .await - .context("Failed to commit changes")?; - - // NOTE: This is needed, so that sqlite "sees" our changes to the table - // without having to reconnect. <2025-02-18> - query!("VACUUM") - .execute(&app.database) - .await - .context("Failed to vacuum database")?; - - Ok(()) - }, - DbVersion::One, - ) - .await?; - Box::pin(Self::Zero.update(app)).await + Self::Empty => { + make_upgrade! {app, Self::Empty, Self::Zero, "./sql/0_Empty_to_Zero.sql"} } - DbVersion::Zero => { - add_error_context( - async { - let mut tx = app - .database - .begin() - .await - .context("Failed to start transaction")?; - debug!("Migrate: Zero -> One"); - - sqlx::raw_sql(include_str!("./sql/01_zero_to_one.sql")) - .execute(&mut *tx) - .await - .context("Failed to execute the update sql script")?; - - set_db_version(&mut tx, Some(DbVersion::Zero), DbVersion::One) - .await - .context("Failed to set the new version")?; - - tx.commit() - .await - .context("Failed to commit the update transaction")?; - - // NOTE: This is needed, so that sqlite "sees" our changes to the table - // without having to reconnect. <2025-02-18> - query!("VACUUM") - .execute(&app.database) - .await - .context("Failed to vacuum database")?; - - Ok(()) - }, - DbVersion::Zero, - ) - .await?; + Self::Zero => { + make_upgrade! {app, Self::Zero, Self::One, "./sql/1_Zero_to_One.sql"} + } - Box::pin(Self::One.update(app)).await + Self::One => { + make_upgrade! {app, Self::One, Self::Two, "./sql/2_One_to_Two.sql"} } - DbVersion::One => { - add_error_context( - async { - let mut tx = app - .database - .begin() - .await - .context("Failed to start the update transaction")?; - debug!("Migrate: One -> Two"); - - sqlx::raw_sql(include_str!("./sql/02_one_to_two.sql")) - .execute(&mut *tx) - .await - .context("Failed to run the update sql script")?; - - set_db_version(&mut tx, Some(DbVersion::One), DbVersion::Two) - .await - .context("Failed to set the new version")?; - - tx.commit() - .await - .context("Failed to commit the update transaction")?; - - // NOTE: This is needed, so that sqlite "sees" our changes to the table - // without having to reconnect. <2025-02-18> - query!("VACUUM") - .execute(&app.database) - .await - .context("Failed to vacuum database")?; - - Ok(()) - }, - DbVersion::One, - ) - .await?; + Self::Two => { + make_upgrade! {app, Self::Two, Self::Three, "./sql/3_Two_to_Three.sql"} + } - Box::pin(Self::Two.update(app)) - .await - .context("Failed to update to version: Three") + Self::Three => { + make_upgrade! {app, Self::Three, Self::Four, "./sql/4_Three_to_Four.sql"} + } + + Self::Four => { + make_upgrade! {app, Self::Four, Self::Five, "./sql/5_Four_to_Five.sql"} } // This is the current_version - DbVersion::Two => { + Self::Five => { assert_eq!(self, CURRENT_VERSION); assert_eq!(self, get_version(app).await?); Ok(()) @@ -263,9 +242,10 @@ fn get_current_date() -> i64 { /// /// # Panics /// Only if internal assertions fail. -pub async fn get_version(app: &App) -> Result<DbVersion> { +pub(crate) async fn get_version(app: &App) -> Result<DbVersion> { get_version_db(&app.database).await } + /// Return the current database version. /// /// In contrast to the [`get_version`] function, this function does not @@ -273,13 +253,19 @@ pub async fn get_version(app: &App) -> Result<DbVersion> { /// /// # Panics /// Only if internal assertions fail. -pub async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { +pub(crate) async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { let version_table_exists = { let query = query!( - "SELECT 1 as result FROM sqlite_master WHERE type = 'table' AND name = 'version'" + " + SELECT 1 as result + FROM sqlite_master + WHERE type = 'table' + AND name = 'version' + " ) .fetch_optional(pool) .await?; + if let Some(output) = query { assert_eq!(output.result, 1); true @@ -287,13 +273,16 @@ pub async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { false } }; + if !version_table_exists { return Ok(DbVersion::Empty); } let current_version = query!( " - SELECT namespace, number FROM version WHERE valid_to IS NULL; + SELECT namespace, number + FROM version + WHERE valid_to IS NULL; " ) .fetch_one(pool) @@ -303,7 +292,7 @@ pub async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { DbVersion::from_db(current_version.number, current_version.namespace.as_str()) } -pub async fn migrate_db(app: &App) -> Result<()> { +pub(crate) async fn migrate_db(app: &App) -> Result<()> { let current_version = get_version(app) .await .context("Failed to determine initial version")?; diff --git a/yt/src/storage/migrate/sql/00_empty_to_zero.sql b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql index d703bfc..d703bfc 100644 --- a/yt/src/storage/migrate/sql/00_empty_to_zero.sql +++ b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql diff --git a/yt/src/storage/migrate/sql/01_zero_to_one.sql b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql index da9315b..da9315b 100644 --- a/yt/src/storage/migrate/sql/01_zero_to_one.sql +++ b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql diff --git a/yt/src/storage/migrate/sql/02_one_to_two.sql b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql index 806de07..806de07 100644 --- a/yt/src/storage/migrate/sql/02_one_to_two.sql +++ b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql diff --git a/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql new file mode 100644 index 0000000..b33f849 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql @@ -0,0 +1,85 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + + +-- 1. Create new table +-- 2. Copy data +-- 3. Drop old table +-- 4. Rename new into old + +-- remove the original TRANSACTION +COMMIT TRANSACTION; + +-- tweak config +PRAGMA foreign_keys=OFF; + +-- start your own TRANSACTION +BEGIN TRANSACTION; + +CREATE TABLE videos_new ( + cache_path TEXT UNIQUE CHECK (CASE + WHEN cache_path IS NOT NULL THEN status == 2 + ELSE 1 + END), + description TEXT, + duration REAL, + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + last_status_change INTEGER NOT NULL, + parent_subscription_name TEXT, + priority INTEGER NOT NULL DEFAULT 0, + publish_date INTEGER, + status INTEGER NOT NULL DEFAULT 0 CHECK (status IN (0, 1, 2, 3, 4, 5) AND + CASE + WHEN status == 2 THEN cache_path IS NOT NULL + WHEN status != 2 THEN cache_path IS NULL + ELSE 1 + END), + thumbnail_url TEXT, + title TEXT NOT NULL, + url TEXT UNIQUE NOT NULL, + is_focused INTEGER UNIQUE DEFAULT NULL CHECK (CASE + WHEN is_focused IS NOT NULL THEN is_focused == 1 + ELSE 1 + END), + watch_progress INTEGER NOT NULL DEFAULT 0 CHECK (watch_progress <= duration) +) STRICT; + +INSERT INTO videos_new SELECT + videos.cache_path, + videos.description, + videos.duration, + videos.extractor_hash, + videos.last_status_change, + videos.parent_subscription_name, + videos.priority, + videos.publish_date, + videos.status, + videos.thumbnail_url, + videos.title, + videos.url, + dummy.is_focused, + videos.watch_progress +FROM videos, (SELECT NULL AS is_focused) AS dummy; + +DROP TABLE videos; + +ALTER TABLE videos_new RENAME TO videos; + +-- check foreign key constraint still upholding. +PRAGMA foreign_key_check; + +-- commit your own TRANSACTION +COMMIT TRANSACTION; + +-- rollback all config you setup before. +PRAGMA foreign_keys=ON; + +-- start a new TRANSACTION to let migrator commit it. +BEGIN TRANSACTION; diff --git a/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql b/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql new file mode 100644 index 0000000..9c283a1 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql @@ -0,0 +1,24 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +ALTER TABLE videos +ADD COLUMN subtitle_langs TEXT; + +ALTER TABLE videos +ADD COLUMN playback_speed REAL CHECK (playback_speed >= 0); + +UPDATE videos + SET playback_speed = video_options.playback_speed, + subtitle_langs = video_options.subtitle_langs + FROM video_options + WHERE videos.extractor_hash = video_options.extractor_hash; + + +DROP TABLE video_options; diff --git a/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql b/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql new file mode 100644 index 0000000..6c4b7cc --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql @@ -0,0 +1,15 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + + +CREATE TABLE txn_log ( + timestamp INTEGER NOT NULL, + operation TEXT NOT NULL +) STRICT; diff --git a/yt/src/storage/mod.rs b/crates/yt/src/storage/mod.rs index 8653eb3..6dcff74 100644 --- a/yt/src/storage/mod.rs +++ b/crates/yt/src/storage/mod.rs @@ -9,6 +9,6 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -pub mod subscriptions; -pub mod video_database; -pub mod migrate; +pub(crate) mod db; +pub(crate) mod migrate; +pub(crate) mod notify; diff --git a/yt/src/storage/video_database/notify.rs b/crates/yt/src/storage/notify.rs index b55c00a..e0ee4e9 100644 --- a/yt/src/storage/video_database/notify.rs +++ b/crates/yt/src/storage/notify.rs @@ -26,7 +26,7 @@ use tokio::task; /// This functions registers a watcher for the database and only returns once a write was /// registered for the database. -pub async fn wait_for_db_write(app: &App) -> Result<()> { +pub(crate) async fn wait_for_db_write(app: &App) -> Result<()> { let db_path: PathBuf = app.config.paths.database_path.clone(); task::spawn_blocking(move || wait_for_db_write_sync(&db_path)).await? } @@ -53,7 +53,7 @@ fn wait_for_db_write_sync(db_path: &Path) -> Result<()> { } /// This functions registers a watcher for the cache path and returns once a file was removed -pub async fn wait_for_cache_reduction(app: &App) -> Result<()> { +pub(crate) async fn wait_for_cache_reduction(app: &App) -> Result<()> { let download_directory: PathBuf = app.config.paths.download_dir.clone(); task::spawn_blocking(move || wait_for_cache_reduction_sync(&download_directory)).await? } diff --git a/yt/src/version/mod.rs b/crates/yt/src/version/mod.rs index 05d85e0..b12eadd 100644 --- a/yt/src/version/mod.rs +++ b/crates/yt/src/version/mod.rs @@ -8,27 +8,13 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use std::process::Command; - use anyhow::{Context, Result}; use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; +use yt_dlp::options::YoutubeDLOptions; use crate::{config::Config, storage::migrate::get_version_db}; -fn get_cmd_version(cmd: &str) -> Result<String> { - let out = String::from_utf8( - Command::new(cmd) - .arg("--version") - .output() - .with_context(|| format!("Failed to run `{cmd} --version`"))? - .stdout, - ) - .context("Failed to interpret output as utf8")?; - - Ok(out.trim().to_owned()) -} - -pub async fn show(config: &Config) -> Result<()> { +pub(crate) async fn show(config: &Config) -> Result<()> { let db_version = { let options = SqliteConnectOptions::new() .filename(&config.paths.database_path) @@ -44,17 +30,20 @@ pub async fn show(config: &Config) -> Result<()> { .context("Failed to determine database version")? }; - // TODO(@bpeetz): Use `pyo3`'s build in mechanism instead of executing the python CLI <2025-02-21> - let python_version = get_cmd_version("python")?; - let yt_dlp_version = get_cmd_version("yt-dlp")?; + let (yt_dlp, python) = { + let yt_dlp = YoutubeDLOptions::new().build()?; + yt_dlp.version()? + }; + + let python = python.replace('\n', " "); println!( "{}: {} db version: {db_version} -python: {python_version} -yt-dlp: {yt_dlp_version}", +yt-dlp: {yt_dlp} +python: {python}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"), ); diff --git a/crates/yt/src/videos/format_video.rs b/crates/yt/src/videos/format_video.rs new file mode 100644 index 0000000..6598780 --- /dev/null +++ b/crates/yt/src/videos/format_video.rs @@ -0,0 +1,133 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use colors::Colorize; + +use crate::{app::App, output::format_text, storage::db::video::Video, videos::RenderWithApp}; + +impl Video { + pub(crate) async fn to_info_display( + &self, + app: &App, + format: Option<String>, + ) -> Result<String> { + let cache_path = self.cache_path_fmt().to_string(app); + let description = self.description_fmt().to_string(app); + let duration = self.duration_fmt().to_string(app); + let extractor_hash = self.extractor_hash_fmt(app).await?.to_string(app); + let in_playlist = self.in_playlist_fmt().to_string(app); + let last_status_change = self.last_status_change_fmt().to_string(app); + let parent_subscription_name = self.parent_subscription_name_fmt().to_string(app); + let priority = self.priority_fmt().to_string(app); + let publish_date = self.publish_date_fmt().to_string(app); + let status = self.status_fmt().to_string(app); + let thumbnail_url = self.thumbnail_url_fmt().to_string(app); + let title = self.title_fmt().to_string(app); + let url = self.url_fmt().to_string(app); + let video_options = self.video_options_fmt(app).to_string(app); + + let watched_percentage_fmt = { + if let Some(percent) = self.watch_progress_percent_fmt() { + format!(" (watched: {})", percent.to_string(app)) + } else { + format!(" {}", self.watch_progress_fmt().to_string(app)) + } + }; + + let options = video_options.to_string(); + let options = options.trim(); + let description = format_text(description.to_string().as_str(), None); + + let string = if let Some(format) = format { + format + .replace("{title}", &title) + .replace("{extractor_hash}", &extractor_hash) + .replace("{cache_path}", &cache_path) + .replace("{duration}", &duration) + .replace("{watched_percentage_fmt}", &watched_percentage_fmt) + .replace("{parent_subscription_name}", &parent_subscription_name) + .replace("{priority}", &priority) + .replace("{publish_date}", &publish_date) + .replace("{status}", &status) + .replace("{last_status_change}", &last_status_change) + .replace("{in_playlist}", &in_playlist) + .replace("{thumbnail_url}", &thumbnail_url) + .replace("{url}", &url) + .replace("{options}", options) + .replace("{description}", &description) + } else { + format!( + "\ +{title} ({extractor_hash}) +| -> {cache_path} +| -> {duration}{watched_percentage_fmt} +| -> {parent_subscription_name} +| -> priority: {priority} +| -> {publish_date} +| -> status: {status} since {last_status_change} ({in_playlist}) +| -> {thumbnail_url} +| -> {url} +| -> options: {options} +{description}\n", + ) + }; + Ok(string) + } + + pub(crate) async fn to_line_display( + &self, + app: &App, + format: Option<String>, + ) -> Result<String> { + let status = self.status_fmt().to_string(app); + let extractor_hash = self.extractor_hash_fmt(app).await?.to_string(app); + let title = self.title_fmt().to_string(app); + let publish_date = self.publish_date_fmt().to_string(app); + let parent_subscription_name = self.parent_subscription_name_fmt().to_string(app); + let duration = self.duration_fmt().to_string(app); + let url = self.url_fmt().to_string(app); + + let f = if let Some(format) = format { + format + .replace("{status}", &status) + .replace("{extractor_hash}", &extractor_hash) + .replace("{title}", &title) + .replace("{publish_date}", &publish_date) + .replace("{parent_subscription_name}", &parent_subscription_name) + .replace("{duration}", &duration) + .replace("{url}", &url) + } else { + format!( + "{status} {extractor_hash} {title} {publish_date} {parent_subscription_name} {duration}" + ) + }; + + Ok(f) + } + + pub(crate) async fn to_select_file_display(&self, app: &App) -> Result<String> { + let f = format!( + r#"{}{} {} "{}" "{}" "{}" "{}" "{}"{}"#, + self.status_fmt().render(false), + self.video_options_fmt(app).render(false), + self.extractor_hash_fmt(app).await?.render(false), + self.title_fmt().render(false), + self.publish_date_fmt().render(false), + self.parent_subscription_name_fmt().render(false), + self.duration_fmt().render(false), + self.url_fmt().render(false), + '\n' + ); + + Ok(f) + } +} diff --git a/crates/yt/src/videos/mod.rs b/crates/yt/src/videos/mod.rs new file mode 100644 index 0000000..c2f01fa --- /dev/null +++ b/crates/yt/src/videos/mod.rs @@ -0,0 +1,213 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::Write; + +use anyhow::{Context, Result}; +use colors::{Colorize, IntoCanvas}; +use url::Url; + +use crate::{ + app::App, + select::duration::MaybeDuration, + storage::db::video::{TimeStamp, Video, VideoStatus}, +}; + +pub(crate) mod format_video; + +macro_rules! get { + ($value:expr, $key:ident, $name:expr, $code:tt) => { + if let Some(value) = &$value.$key { + $code(value) + } else { + concat!("[No ", $name, "]").to_owned() + } + }; +} + +pub(crate) trait RenderWithApp: Colorize { + fn to_string(self, app: &App) -> String { + self.render(app.config.global.display_colors) + } +} +impl<C: Colorize> RenderWithApp for C {} + +impl Video { + #[must_use] + pub(crate) fn cache_path_fmt(&self) -> impl Colorize { + let cache_path = if let VideoStatus::Cached { + cache_path, + is_focused: _, + } = &self.status + { + cache_path.to_string_lossy().to_string() + } else { + "[No Cache Path]".to_owned() + }; + + cache_path.blue().bold() + } + + #[must_use] + pub(crate) fn description_fmt(&self) -> impl Colorize { + get!( + self, + description, + "Description", + (|value: &str| value.to_owned()) + ) + .into_canvas() + } + + #[must_use] + pub(crate) fn duration_fmt(&self) -> impl Colorize { + self.duration.cyan().bold() + } + + #[must_use] + pub(crate) fn watch_progress_fmt(&self) -> impl Colorize { + MaybeDuration::from_std(self.watch_progress).cyan().bold() + } + #[must_use] + pub(crate) fn watch_progress_percent_fmt(&self) -> Option<impl Colorize> { + self.duration.as_secs_f64().map(|duration| { + let watch_progress = self.watch_progress.as_secs_f64(); + + (format!("{:0.0}%", (watch_progress / duration) * 100.0)).into_canvas() + }) + } + + pub(crate) async fn extractor_hash_fmt(&self, app: &App) -> Result<impl Colorize> { + let hash = self + .extractor_hash + .as_short_hash(app) + .await + .with_context(|| { + format!( + "Failed to format extractor hash, whilst formatting video: '{}'", + self.title + ) + })?; + + Ok(hash.purple().bold().italic()) + } + + #[must_use] + pub(crate) fn in_playlist_fmt(&self) -> impl Colorize { + let output = match &self.status { + VideoStatus::Pick + | VideoStatus::Watch + | VideoStatus::Watched + | VideoStatus::Drop + | VideoStatus::Dropped => "Not in the playlist", + VideoStatus::Cached { is_focused, .. } => { + if *is_focused { + "In the playlist and focused" + } else { + "In the playlist" + } + } + }; + output.yellow().italic() + } + #[must_use] + pub(crate) fn last_status_change_fmt(&self) -> impl Colorize { + self.last_status_change.bright_cyan() + } + + #[must_use] + pub(crate) fn parent_subscription_name_fmt(&self) -> impl Colorize { + let psn = get!( + self, + parent_subscription_name, + "author", + (|sub: &str| sub.replace('"', "'")) + ); + + psn.bright_magenta() + } + + #[must_use] + pub(crate) fn priority_fmt(&self) -> impl Colorize { + self.priority.into_canvas() + } + + #[must_use] + pub(crate) fn publish_date_fmt(&self) -> impl Colorize { + let date = get!( + self, + publish_date, + "release date", + (|date: &TimeStamp| date.to_string()) + ); + + date.bright_white().bold() + } + + #[must_use] + pub(crate) fn status_fmt(&self) -> impl Colorize { + // TODO: We might support `.trim()`ing that, as the extra whitespace could be bad in the + // selection file. <2024-10-07> + let status = self.status.as_marker().as_command().to_owned(); + + status.red().bold() + } + + #[must_use] + pub(crate) fn thumbnail_url_fmt(&self) -> impl Colorize { + get!( + self, + thumbnail_url, + "thumbnail URL", + (|url: &Url| url.to_string()) + ) + .into_canvas() + } + + #[must_use] + pub(crate) fn title_fmt(&self) -> impl Colorize { + let title = self.title.replace(['"', '„', '”', '“'], "'"); + + title.green().bold() + } + + #[must_use] + pub(crate) fn url_fmt(&self) -> impl Colorize { + let url = self.url.as_str().replace('"', "\\\""); + + url.italic() + } + + pub(crate) fn video_options_fmt(&self, app: &App) -> impl Colorize { + let video_options = { + let mut opts = String::new(); + + if let Some(playback_speed) = self.playback_speed { + if (playback_speed - app.config.select.playback_speed).abs() > f64::EPSILON { + write!(opts, " --playback-speed '{}'", playback_speed).expect("In-memory"); + } + } + + if let Some(subtitle_langs) = &self.subtitle_langs { + if subtitle_langs != &app.config.select.subtitle_langs { + write!(opts, " --subtitle-langs '{}'", subtitle_langs).expect("In-memory"); + } + } + + let opts = opts.trim().to_owned(); + + let opts_white = if opts.is_empty() { "" } else { " " }; + format!("{opts_white}{opts}") + }; + + video_options.bright_green() + } +} diff --git a/crates/yt/src/yt_dlp/mod.rs b/crates/yt/src/yt_dlp/mod.rs new file mode 100644 index 0000000..eaa80a1 --- /dev/null +++ b/crates/yt/src/yt_dlp/mod.rs @@ -0,0 +1,253 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{borrow::ToOwned, str::FromStr, time::Duration}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use futures::{FutureExt, future::BoxFuture}; +use log::{error, warn}; +use serde_json::json; +use tokio::{fs, io}; +use url::Url; +use yt_dlp::{ + YoutubeDL, info_json::InfoJson, json_cast, json_get, json_try_get, options::YoutubeDLOptions, +}; + +use crate::{ + app::App, + select::duration::MaybeDuration, + shared::bytes::Bytes, + storage::db::{ + extractor_hash::ExtractorHash, + subscription::Subscription, + video::{Priority, TimeStamp, Video, VideoStatus}, + }, +}; + +pub(crate) fn yt_dlp_opts_updating(max_backlog: usize) -> Result<YoutubeDL> { + Ok(YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", max_backlog) + .set("noplaylist", false) + .set( + "extractor_args", + json! {{"youtubetab": {"approximate_date": [""]}}}, + ) + // // TODO: This also removes unlisted and other stuff. Find a good way to remove the + // // members-only videos from the feed. <2025-04-17> + // .set("match-filter", "availability=public") + .build()?) +} + +impl Video { + pub(crate) fn get_approx_size(&self) -> Result<u64> { + let yt_dlp = { + YoutubeDLOptions::new() + .set("prefer_free_formats", true) + .set("format", "bestvideo[height<=?1080]+bestaudio/best") + .set("fragment_retries", 10) + .set("retries", 10) + .set("getcomments", false) + .set("ignoreerrors", false) + .build() + .context("Failed to instanciate get approx size yt_dlp") + }?; + + let result = yt_dlp + .extract_info(&self.url, false, true) + .with_context(|| format!("Failed to extract video information: '{}'", self.title))?; + + let size = if let Some(filesize) = json_try_get!(result, "filesize", as_u64) { + filesize + } else if let Some(num) = json_try_get!(result, "filesize_approx", as_u64) { + num + } else if let Some(duration) = json_try_get!(result, "duration", as_f64) + && let Some(tbr) = json_try_get!(result, "tbr", as_f64) + { + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let duration = duration.ceil() as u64; + + // TODO: yt_dlp gets this from the format + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let tbr = tbr.ceil() as u64; + + duration * tbr * (1000 / 8) + } else { + let hardcoded_default = Bytes::from_str("250 MiB").expect("This is hardcoded"); + error!( + "Failed to find a filesize for video: {:?} (Using hardcoded value of {})", + self.title, hardcoded_default + ); + hardcoded_default.as_u64() + }; + + Ok(size) + } +} + +impl Video { + #[allow(clippy::too_many_lines)] + pub(crate) fn from_info_json(entry: &InfoJson, sub: Option<&Subscription>) -> Result<Video> { + fn fmt_context(date: &str, extended: Option<&str>) -> String { + let f = format!( + "Failed to parse the `upload_date` of the entry ('{date}'). \ + Expected `YYYY-MM-DD`, has the format changed?" + ); + if let Some(date_string) = extended { + format!("{f}\nThe parsed '{date_string}' can't be turned to a valid UTC date.'") + } else { + f + } + } + + let publish_date = if let Some(date) = json_try_get!(entry, "upload_date", as_str) { + let year: u32 = date + .chars() + .take(4) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let month: u32 = date + .chars() + .skip(4) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let day: u32 = date + .chars() + .skip(4 + 2) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + + let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z"); + Some( + DateTime::<Utc>::from_str(&date_string) + .with_context(|| fmt_context(date, Some(&date_string)))? + .timestamp(), + ) + } else { + warn!( + "The video '{}' lacks it's upload date!", + json_get!(entry, "title", as_str) + ); + None + }; + + let thumbnail_url = match ( + &json_try_get!(entry, "thumbnails", as_array), + &json_try_get!(entry, "thumbnail", as_str), + ) { + (None, None) => None, + (None, Some(thumbnail)) => Some(Url::from_str(thumbnail)?), + + // TODO: The algorithm is not exactly the best <2024-05-28> + (Some(thumbnails), None) => { + if let Some(thumbnail) = thumbnails.first() { + Some(Url::from_str(json_get!( + json_cast!(thumbnail, as_object), + "url", + as_str + ))?) + } else { + None + } + } + (Some(_), Some(thumnail)) => Some(Url::from_str(thumnail)?), + }; + + let url = { + let smug_url: Url = json_get!(entry, "webpage_url", as_str).parse()?; + // TODO(@bpeetz): We should probably add this? <2025-06-14> + // if '#__youtubedl_smuggle' not in smug_url: + // return smug_url, default + // url, _, sdata = smug_url.rpartition('#') + // jsond = urllib.parse.parse_qs(sdata)['__youtubedl_smuggle'][0] + // data = json.loads(jsond) + // return url, data + + smug_url + }; + + let extractor_hash = ExtractorHash::from_info_json(entry); + + let subscription_name = if let Some(sub) = sub { + Some(sub.name.clone()) + } else if let Some(uploader) = json_try_get!(entry, "uploader", as_str) { + if json_try_get!(entry, "webpage_url_domain", as_str) == Some("youtube.com") { + Some(format!("{uploader} - Videos")) + } else { + Some(uploader.to_owned()) + } + } else { + None + }; + + let video = Video { + description: json_try_get!(entry, "description", as_str).map(ToOwned::to_owned), + duration: MaybeDuration::from_maybe_secs_f64(json_try_get!(entry, "duration", as_f64)), + extractor_hash, + last_status_change: TimeStamp::from_now(), + parent_subscription_name: subscription_name, + priority: Priority::default(), + publish_date: publish_date.map(TimeStamp::from_secs), + status: VideoStatus::Pick, + thumbnail_url, + title: json_get!(entry, "title", as_str).to_owned(), + url, + watch_progress: Duration::default(), + playback_speed: None, + subtitle_langs: None, + }; + Ok(video) + } +} + +pub(crate) async fn get_current_cache_allocation(app: &App) -> Result<Bytes> { + fn dir_size(mut dir: fs::ReadDir) -> BoxFuture<'static, Result<Bytes>> { + async move { + let mut acc = 0; + while let Some(entry) = dir.next_entry().await? { + let size = match entry.metadata().await? { + data if data.is_dir() => { + let path = entry.path(); + let read_dir = fs::read_dir(path).await?; + + dir_size(read_dir).await?.as_u64() + } + data => data.len(), + }; + acc += size; + } + Ok(Bytes::new(acc)) + } + .boxed() + } + + let read_dir_result = match fs::read_dir(&app.config.paths.download_dir).await { + Ok(ok) => ok, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => { + unreachable!("The download dir should always be created in the config finalizers."); + } + err => Err(io::Error::from(err)).with_context(|| { + format!( + "Failed to get dir size of download dir at: '{}'", + &app.config.paths.download_dir.display() + ) + })?, + }, + }; + + dir_size(read_dir_result).await +} diff --git a/crates/yt/tests/_testenv/init.rs b/crates/yt/tests/_testenv/init.rs new file mode 100644 index 0000000..5970c7c --- /dev/null +++ b/crates/yt/tests/_testenv/init.rs @@ -0,0 +1,136 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + env, + fs::{self, OpenOptions}, + io::{self, Write}, + path::PathBuf, +}; + +use crate::{_testenv::Paths, testenv::TestEnv}; + +fn target_dir() -> PathBuf { + // Tests exe is in target/debug/deps, the *yt* exe is in target/debug + env::current_exe() + .expect("./target/debug/deps/yt-*") + .parent() + .expect("./target/debug/deps") + .parent() + .expect("./target/debug") + .parent() + .expect("./target") + .to_path_buf() +} + +fn test_dir(name: &'static str) -> PathBuf { + target_dir().join("tests").join(name) +} + +fn prepare_files_and_dirs(name: &'static str) -> io::Result<Paths> { + let test_dir = test_dir(name); + + fs::create_dir_all(&test_dir)?; + + let db_path = test_dir.join("database.sqlite"); + let last_selection_path = test_dir.join("last_selection"); + let config_path = test_dir.join("config.toml"); + let download_dir = test_dir.join("download"); + + { + // Remove all files, that are not the download dir. + for entry in fs::read_dir(test_dir).expect("Works") { + let entry = entry.unwrap(); + let entry_ft = entry.file_type().unwrap(); + + if entry_ft.is_dir() && entry.file_name() == "download" { + continue; + } + + if entry_ft.is_dir() { + fs::remove_dir_all(entry.path()).unwrap(); + } else { + fs::remove_file(entry.path()).unwrap(); + } + } + } + + fs::create_dir_all(&download_dir)?; + + { + let mut config_file = OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(&config_path)?; + + writeln!( + config_file, + r#"[paths] +download_dir = "{}" +mpv_config_path = "/dev/null" +mpv_input_path = "/dev/null" +database_path = "{}" +last_selection_path = "{}" + +[download] +max_cache_size = "100GiB" + +[update] +max_backlog = 1 +"#, + download_dir.display(), + db_path.display(), + last_selection_path.display(), + )?; + config_file.flush()?; + } + + Ok(Paths { + db: db_path, + last_selection: last_selection_path, + config: config_path, + download_dir, + }) +} + +/// Find the *yt* executable. +fn find_yt_exe() -> PathBuf { + let target = target_dir().join("debug"); + + let exe_name = if cfg!(windows) { "yt.exe" } else { "yt" }; + + target.join(exe_name) +} + +impl TestEnv { + pub(crate) fn new(name: &'static str) -> TestEnv { + let yt_exe = find_yt_exe(); + let test_dir = test_dir(name); + + let paths = prepare_files_and_dirs(name).expect("config dir"); + + TestEnv { + name, + yt_exe, + test_dir, + paths, + spawned_childs: vec![], + } + } +} + +impl Drop for TestEnv { + fn drop(&mut self) { + for child in &mut self.spawned_childs { + drop(child.kill()); + } + } +} diff --git a/crates/yt/tests/_testenv/mod.rs b/crates/yt/tests/_testenv/mod.rs new file mode 100644 index 0000000..38d1f0a --- /dev/null +++ b/crates/yt/tests/_testenv/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! This code was taken from *fd* at 30-06-2025. + +use std::{path::PathBuf, process}; + +mod init; +mod run; +pub(crate) mod util; + +/// Environment for the integration tests. +pub(crate) struct TestEnv { + pub(crate) name: &'static str, + pub(crate) yt_exe: PathBuf, + pub(crate) test_dir: PathBuf, + + pub(crate) paths: Paths, + + pub(crate) spawned_childs: Vec<process::Child>, +} + +pub(crate) struct Paths { + pub(crate) db: PathBuf, + pub(crate) last_selection: PathBuf, + pub(crate) config: PathBuf, + pub(crate) download_dir: PathBuf, +} diff --git a/crates/yt/tests/_testenv/run.rs b/crates/yt/tests/_testenv/run.rs new file mode 100644 index 0000000..578d823 --- /dev/null +++ b/crates/yt/tests/_testenv/run.rs @@ -0,0 +1,183 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::HashMap, + process::{self, Stdio}, +}; + +use colors::{Colorize, IntoCanvas}; + +use crate::testenv::TestEnv; + +/// Format an error message for when *yt* did not exit successfully. +fn format_exit_error(args: &[&str], output: &process::Output) -> String { + format!( + "`yt {}` did not exit successfully.\nstdout:\n---\n{}---\nstderr:\n---\n{}---", + args.join(" "), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// Format an error message for when the output of *yt* did not match the expected output. +fn format_output_error(args: &[&str], expected: &str, actual: &str) -> String { + fn normalize_str(input: &str) -> &str { + if input.is_empty() { "<Empty>" } else { input } + } + + let expected = normalize_str(expected); + let actual = normalize_str(actual); + + format!( + concat!( + "`yt {}` did not produce the expected output.\n", + "expected:\n---\n{}\n---\n and actual:\n---\n{}\n---\n" + ), + args.join(" "), + expected, + actual + ) +} + +/// Normalize the output for comparison. +fn normalize_output(s: &str, trim_start: bool, normalize_line: bool) -> String { + // Split into lines and normalize separators. + let mut lines = s + .replace('\0', "NULL\n") + .lines() + .map(|line| { + let line = if trim_start { line.trim_start() } else { line }; + let line = line.replace('/', std::path::MAIN_SEPARATOR_STR); + if normalize_line { + let mut words: Vec<_> = line.split_whitespace().collect(); + words.sort_unstable(); + return words.join(" "); + } + line + }) + .collect::<Vec<_>>(); + + lines.sort(); + lines.join("\n") +} + +impl TestEnv { + /// Assert that calling *yt* in the specified path under the root working directory, + /// and with the specified arguments produces the expected output. + pub(crate) fn assert_output(&self, args: &[&str], expected: &str) { + let expected = normalize_output(expected, true, false); + let actual = self.run(args); + + assert!( + expected == actual, + "{}", + format_output_error(args, &expected, &actual) + ); + } + + /// Run *yt* once with the first args, pipe the output of this command to *yt* with the second + /// args and return the normalized output. + pub(crate) fn run_piped(&self, first_args: &[&str], second_args: &[&str]) -> String { + let mut first_cmd = self.prepare_yt(first_args); + let mut second_cmd = self.prepare_yt(second_args); + + first_cmd.stdout(Stdio::piped()); + let mut first_child = first_cmd.spawn().expect("yt spawn"); + + second_cmd.stdin(first_child.stdout.take().expect("Was set")); + + let first_output = first_child.wait_with_output().expect("yt run"); + assert!( + first_output.status.success(), + "{}", + format_exit_error(first_args, &first_output) + ); + + Self::finalize_cmd(second_cmd, second_args) + } + + /// Run *yt*, with the given args. + /// Returns the normalized stdout output. + pub(crate) fn run(&self, args: &[&str]) -> String { + let cmd = self.prepare_yt(args); + Self::finalize_cmd(cmd, args) + } + + /// Run *yt*, with the given args and environment variables. + /// Returns the normalized stdout output. + pub(crate) fn run_env(&self, args: &[&str], env: &HashMap<&str, &str>) -> String { + let mut cmd = self.prepare_yt(args); + cmd.envs(env.iter()); + Self::finalize_cmd(cmd, args) + } + + /// Run *yt*, with the given args and fork into the background. + /// Returns a sender for the lines on stdout. + pub(crate) fn run_background(&mut self, args: &[&str]) -> process::ChildStdout { + let mut cmd = self.prepare_yt(args); + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::inherit()); + + // The whole point of this function, is calling a command and keep it running for the whole + // programs run-time. + // + // And we provide a clean-up mechanism. + #[allow(clippy::zombie_processes)] + let mut child = cmd.spawn().expect("yt spawn"); + + let stdout = child.stdout.take().expect("Was piped"); + + self.spawned_childs.push(child); + + stdout + } + + fn finalize_cmd(mut cmd: process::Command, args: &[&str]) -> String { + let child = cmd.spawn().expect("yt spawn"); + let output = child.wait_with_output().expect("yt output"); + + assert!( + output.status.success(), + "{}", + format_exit_error(args, &output) + ); + + normalize_output(&String::from_utf8_lossy(&output.stdout), false, false) + } + + fn prepare_yt(&self, args: &[&str]) -> process::Command { + let mut cmd = process::Command::new(&self.yt_exe); + + cmd.current_dir(&self.test_dir); + + cmd.args([ + "-v", + "--color", + "false", + "--config-path", + self.paths.config.to_str().unwrap(), + ]); + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + cmd.args(args); + + eprintln!( + "{} `yt {}`", + self.name.blue().italic().render(true), + args.join(" ") + ); + + cmd + } +} diff --git a/crates/yt/tests/_testenv/util.rs b/crates/yt/tests/_testenv/util.rs new file mode 100644 index 0000000..6633fbf --- /dev/null +++ b/crates/yt/tests/_testenv/util.rs @@ -0,0 +1,371 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::_testenv::TestEnv; + +use std::{ + collections::HashMap, + fs::{OpenOptions, Permissions}, + io::Write, + os::unix::fs::PermissionsExt, + path::PathBuf, +}; + +pub(crate) fn get_first_hash(env: &TestEnv) -> String { + let output = env.run(&["videos", "ls", "--format", "{extractor_hash}"]); + + let first_hash = output.lines().next().unwrap(); + first_hash.to_owned() +} + +#[derive(Clone, Copy)] +pub(crate) enum Subscription { + Tagesschau, +} + +impl Subscription { + const fn as_url(self) -> &'static str { + match self { + Subscription::Tagesschau => { + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU" + } + } + } + + const fn as_name(self) -> &'static str { + match self { + Subscription::Tagesschau => "Tagesschau", + } + } +} + +/// Provide the given number of videos. +pub(crate) fn provide_videos(env: &TestEnv, sub: Subscription, num: u8) { + add_sub(env, sub); + update_sub(env, num, sub); +} + +pub(crate) fn add_sub(env: &TestEnv, sub: Subscription) { + env.run(&[ + "subs", + "add", + sub.as_url(), + "--name", + sub.as_name(), + "--no-check", + ]); +} + +pub(crate) fn update_sub(env: &TestEnv, num_of_videos: u8, sub: Subscription) { + let num_of_videos: &str = u8_to_char(num_of_videos); + + env.run(&["update", "--max-backlog", num_of_videos, sub.as_name()]); +} + +pub(crate) fn run_select(env: &TestEnv, sed_regex: &str) { + let mut map = HashMap::new(); + + let command = make_command( + env, + format!(r#"sed --in-place '{sed_regex}' "$1""#).as_str(), + ); + map.insert("EDITOR", command.to_str().unwrap()); + + env.run_env(&["select", "file", "--done"], &map); +} + +fn make_command(env: &TestEnv, shell_command: &str) -> PathBuf { + let command_path = env.test_dir.join("command.sh"); + + { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&command_path) + .unwrap(); + + writeln!(file, "#!/usr/bin/env sh").unwrap(); + + file.write_all(shell_command.as_bytes()).unwrap(); + file.flush().unwrap(); + + { + let perms = Permissions::from_mode(0o0700); + file.set_permissions(perms).unwrap(); + } + } + + command_path +} + +// Char conversion {{{ +#[allow(clippy::too_many_lines)] +const fn u8_to_char(input: u8) -> &'static str { + match input { + 0 => "0", + 1 => "1", + 2 => "2", + 3 => "3", + 4 => "4", + 5 => "5", + 6 => "6", + 7 => "7", + 8 => "8", + 9 => "9", + 10 => "10", + 11 => "11", + 12 => "12", + 13 => "13", + 14 => "14", + 15 => "15", + 16 => "16", + 17 => "17", + 18 => "18", + 19 => "19", + 20 => "20", + 21 => "21", + 22 => "22", + 23 => "23", + 24 => "24", + 25 => "25", + 26 => "26", + 27 => "27", + 28 => "28", + 29 => "29", + 30 => "30", + 31 => "31", + 32 => "32", + 33 => "33", + 34 => "34", + 35 => "35", + 36 => "36", + 37 => "37", + 38 => "38", + 39 => "39", + 40 => "40", + 41 => "41", + 42 => "42", + 43 => "43", + 44 => "44", + 45 => "45", + 46 => "46", + 47 => "47", + 48 => "48", + 49 => "49", + 50 => "50", + 51 => "51", + 52 => "52", + 53 => "53", + 54 => "54", + 55 => "55", + 56 => "56", + 57 => "57", + 58 => "58", + 59 => "59", + 60 => "60", + 61 => "61", + 62 => "62", + 63 => "63", + 64 => "64", + 65 => "65", + 66 => "66", + 67 => "67", + 68 => "68", + 69 => "69", + 70 => "70", + 71 => "71", + 72 => "72", + 73 => "73", + 74 => "74", + 75 => "75", + 76 => "76", + 77 => "77", + 78 => "78", + 79 => "79", + 80 => "80", + 81 => "81", + 82 => "82", + 83 => "83", + 84 => "84", + 85 => "85", + 86 => "86", + 87 => "87", + 88 => "88", + 89 => "89", + 90 => "90", + 91 => "91", + 92 => "92", + 93 => "93", + 94 => "94", + 95 => "95", + 96 => "96", + 97 => "97", + 98 => "98", + 99 => "99", + 100 => "100", + 101 => "101", + 102 => "102", + 103 => "103", + 104 => "104", + 105 => "105", + 106 => "106", + 107 => "107", + 108 => "108", + 109 => "109", + 110 => "110", + 111 => "111", + 112 => "112", + 113 => "113", + 114 => "114", + 115 => "115", + 116 => "116", + 117 => "117", + 118 => "118", + 119 => "119", + 120 => "120", + 121 => "121", + 122 => "122", + 123 => "123", + 124 => "124", + 125 => "125", + 126 => "126", + 127 => "127", + 128 => "128", + 129 => "129", + 130 => "130", + 131 => "131", + 132 => "132", + 133 => "133", + 134 => "134", + 135 => "135", + 136 => "136", + 137 => "137", + 138 => "138", + 139 => "139", + 140 => "140", + 141 => "141", + 142 => "142", + 143 => "143", + 144 => "144", + 145 => "145", + 146 => "146", + 147 => "147", + 148 => "148", + 149 => "149", + 150 => "150", + 151 => "151", + 152 => "152", + 153 => "153", + 154 => "154", + 155 => "155", + 156 => "156", + 157 => "157", + 158 => "158", + 159 => "159", + 160 => "160", + 161 => "161", + 162 => "162", + 163 => "163", + 164 => "164", + 165 => "165", + 166 => "166", + 167 => "167", + 168 => "168", + 169 => "169", + 170 => "170", + 171 => "171", + 172 => "172", + 173 => "173", + 174 => "174", + 175 => "175", + 176 => "176", + 177 => "177", + 178 => "178", + 179 => "179", + 180 => "180", + 181 => "181", + 182 => "182", + 183 => "183", + 184 => "184", + 185 => "185", + 186 => "186", + 187 => "187", + 188 => "188", + 189 => "189", + 190 => "190", + 191 => "191", + 192 => "192", + 193 => "193", + 194 => "194", + 195 => "195", + 196 => "196", + 197 => "197", + 198 => "198", + 199 => "199", + 200 => "200", + 201 => "201", + 202 => "202", + 203 => "203", + 204 => "204", + 205 => "205", + 206 => "206", + 207 => "207", + 208 => "208", + 209 => "209", + 210 => "210", + 211 => "211", + 212 => "212", + 213 => "213", + 214 => "214", + 215 => "215", + 216 => "216", + 217 => "217", + 218 => "218", + 219 => "219", + 220 => "220", + 221 => "221", + 222 => "222", + 223 => "223", + 224 => "224", + 225 => "225", + 226 => "226", + 227 => "227", + 228 => "228", + 229 => "229", + 230 => "230", + 231 => "231", + 232 => "232", + 233 => "233", + 234 => "234", + 235 => "235", + 236 => "236", + 237 => "237", + 238 => "238", + 239 => "239", + 240 => "240", + 241 => "241", + 242 => "242", + 243 => "243", + 244 => "244", + 245 => "245", + 246 => "246", + 247 => "247", + 248 => "248", + 249 => "249", + 250 => "250", + 251 => "251", + 252 => "252", + 253 => "253", + 254 => "254", + 255 => "255", + } +} +// }}} diff --git a/crates/yt/tests/select/base.rs b/crates/yt/tests/select/base.rs new file mode 100644 index 0000000..24e198b --- /dev/null +++ b/crates/yt/tests/select/base.rs @@ -0,0 +1,50 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, fs}; + +use crate::{ + _testenv::{ + TestEnv, + util::{self, Subscription}, + }, + select::get_videos_in_state, +}; + +#[test] +fn test_base() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, Subscription::Tagesschau, 4); + + let first_hash = &util::get_first_hash(&env); + env.run(&["select", "drop", first_hash]); + + let mut map = HashMap::new(); + map.insert("EDITOR", "true"); + env.run_env(&["select", "file", "--done"], &map); + + fs::remove_file(&env.paths.last_selection).unwrap(); + + env.run_env(&["select", "split", "--done"], &map); + assert_states(&env); + + env.run_env(&["select", "file", "--use-last-selection"], &map); + assert_states(&env); +} + +fn assert_states(env: &TestEnv) { + assert_eq!(get_videos_in_state(env, "Picked"), 3); + assert_eq!(get_videos_in_state(env, "Drop"), 1); + + assert_eq!(get_videos_in_state(env, "Watch"), 0); + assert_eq!(get_videos_in_state(env, "Cached"), 0); + assert_eq!(get_videos_in_state(env, "Watched"), 0); +} diff --git a/crates/yt/tests/select/file.rs b/crates/yt/tests/select/file.rs new file mode 100644 index 0000000..b8bd2b5 --- /dev/null +++ b/crates/yt/tests/select/file.rs @@ -0,0 +1,31 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + _testenv::util::{self, Subscription}, + select::get_videos_in_state, + testenv::TestEnv, +}; + +#[test] +fn test_file() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, Subscription::Tagesschau, 4); + + util::run_select(&env, "s/pick/watch/"); + + assert_eq!(get_videos_in_state(&env, "Picked"), 0); + assert_eq!(get_videos_in_state(&env, "Drop"), 0); + + assert_eq!(get_videos_in_state(&env, "Watch"), 4); + assert_eq!(get_videos_in_state(&env, "Cached"), 0); + assert_eq!(get_videos_in_state(&env, "Watched"), 0); +} diff --git a/crates/yt/tests/select/mod.rs b/crates/yt/tests/select/mod.rs new file mode 100644 index 0000000..d7033f8 --- /dev/null +++ b/crates/yt/tests/select/mod.rs @@ -0,0 +1,25 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::_testenv::TestEnv; + +mod base; +mod file; +mod options; + +fn get_videos_in_state(env: &TestEnv, state: &str) -> usize { + let status = env.run(&[ + "status", + "--format", + format!("{{{}_videos_len}}", state.to_lowercase()).as_str(), + ]); + + status.parse().unwrap() +} diff --git a/crates/yt/tests/select/options.rs b/crates/yt/tests/select/options.rs new file mode 100644 index 0000000..6a0d155 --- /dev/null +++ b/crates/yt/tests/select/options.rs @@ -0,0 +1,51 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{_testenv::util, select::get_videos_in_state, testenv::TestEnv}; + +#[test] +fn test_options() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, util::Subscription::Tagesschau, 1); + + let video_hash = &util::get_first_hash(&env); + env.run(&["select", "watch", video_hash]); + + env.assert_output(&["videos", "info", video_hash, "--format", "{options}"], ""); + + env.run(&[ + "select", + "watch", + video_hash, + "--playback-speed", + "1", + "--subtitle-langs", + "en,de,sv", + ]); + + env.assert_output( + &["videos", "info", video_hash, "--format", "{options}"], + "--playback-speed '1' --subtitle-langs 'en,de,sv'", + ); + + env.run(&["select", "watch", video_hash, "-s", "1.7", "-l", "de"]); + + env.assert_output( + &["videos", "info", video_hash, "--format", "{options}"], + "--playback-speed '1.7' --subtitle-langs 'de'", + ); + + assert_eq!(get_videos_in_state(&env, "Picked"), 0); + assert_eq!(get_videos_in_state(&env, "Drop"), 0); + assert_eq!(get_videos_in_state(&env, "Watch"), 1); + assert_eq!(get_videos_in_state(&env, "Cached"), 0); + assert_eq!(get_videos_in_state(&env, "Watched"), 0); +} diff --git a/crates/yt/tests/subscriptions/import_export/golden.txt b/crates/yt/tests/subscriptions/import_export/golden.txt new file mode 100644 index 0000000..7ed5419 --- /dev/null +++ b/crates/yt/tests/subscriptions/import_export/golden.txt @@ -0,0 +1,2 @@ +Nyheter på lätt svenska: 'https://www.svtplay.se/nyheter-pa-latt-svenska' +tagesschau: 'https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU' diff --git a/crates/yt_dlp/src/python_json_decode_failed.error_msg.license b/crates/yt/tests/subscriptions/import_export/golden.txt.license index 7813eb6..7813eb6 100644 --- a/crates/yt_dlp/src/python_json_decode_failed.error_msg.license +++ b/crates/yt/tests/subscriptions/import_export/golden.txt.license diff --git a/crates/yt/tests/subscriptions/import_export/mod.rs b/crates/yt/tests/subscriptions/import_export/mod.rs new file mode 100644 index 0000000..1156508 --- /dev/null +++ b/crates/yt/tests/subscriptions/import_export/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::testenv::TestEnv; + +#[test] +fn test_import_export() { + let env = TestEnv::new(module_path!()); + + env.run(&[ + "subs", + "add", + "https://www.svtplay.se/nyheter-pa-latt-svenska", + ]); + env.run(&[ + "subs", + "add", + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU", + ]); + + let before = env.run(&["subs", "list"]); + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); + + env.run_piped(&["subs", "export"], &["subs", "import", "--force"]); + + env.assert_output(&["subs", "list"], &before); + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); +} diff --git a/crates/yt/tests/subscriptions/mod.rs b/crates/yt/tests/subscriptions/mod.rs new file mode 100644 index 0000000..0b300c5 --- /dev/null +++ b/crates/yt/tests/subscriptions/mod.rs @@ -0,0 +1,12 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +mod import_export; +mod naming_subscriptions; diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt new file mode 100644 index 0000000..46ede50 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt @@ -0,0 +1,2 @@ +Nyheter: 'https://www.svtplay.se/nyheter-pa-latt-svenska' +Vewn: 'https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU' diff --git a/crates/bytes/Cargo.lock.license b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license index d4d410f..7813eb6 100644 --- a/crates/bytes/Cargo.lock.license +++ b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license @@ -1,6 +1,6 @@ yt - A fully featured command line YouTube client -Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> SPDX-License-Identifier: GPL-3.0-or-later This file is part of Yt. diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs b/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs new file mode 100644 index 0000000..50fe3e4 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs @@ -0,0 +1,33 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::testenv::TestEnv; + +#[test] +fn test_naming_subscriptions() { + let env = TestEnv::new(module_path!()); + + env.run(&[ + "subs", + "add", + "https://www.svtplay.se/nyheter-pa-latt-svenska", + "--name", + "Nyheter", + ]); + env.run(&[ + "subs", + "add", + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU", + "--name", + "Vewn", + ]); + + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); +} diff --git a/crates/yt/tests/tests.rs b/crates/yt/tests/tests.rs new file mode 100644 index 0000000..89c3091 --- /dev/null +++ b/crates/yt/tests/tests.rs @@ -0,0 +1,22 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// Use this, for the background run pids +// #![feature(linux_pidfd)] + +#![allow(unused_crate_dependencies)] + +mod _testenv; +pub(crate) use _testenv as testenv; + +mod select; +mod subscriptions; +mod videos; +mod watch; diff --git a/crates/yt/tests/videos/downloading.rs b/crates/yt/tests/videos/downloading.rs new file mode 100644 index 0000000..f026858 --- /dev/null +++ b/crates/yt/tests/videos/downloading.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{_testenv::util, testenv::TestEnv}; + +#[test] +fn test_downloading() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, util::Subscription::Tagesschau, 1); + + let first_hash = &util::get_first_hash(&env); + env.run(&["select", "watch", first_hash]); + + env.run(&["download"]); + + let usage = get_cache_usage(&env); + assert!(usage > 0.0); + + env.run(&["cache", "clear"]); + + let usage = get_cache_usage(&env); + + #[allow(clippy::float_cmp)] + { + assert_eq!(usage, 0.0); + } +} + +fn get_cache_usage(env: &TestEnv) -> f64 { + let status = env.run(&["status", "--format", "{cache_usage}"]); + + let split: Vec<_> = status.split(' ').collect(); + let usage: f64 = split[0].parse().unwrap(); + let unit = split[1]; + + #[allow(clippy::cast_precision_loss)] + match unit { + "B" => usage * (1024u64.pow(0)) as f64, + "KiB" => usage * (1024u64.pow(1)) as f64, + "MiB" => usage * (1024u64.pow(2)) as f64, + "GiB" => usage * (1024u64.pow(3)) as f64, + other => unreachable!("Unknown unit: {other}"), + } +} diff --git a/crates/yt/tests/videos/mod.rs b/crates/yt/tests/videos/mod.rs new file mode 100644 index 0000000..6a80761 --- /dev/null +++ b/crates/yt/tests/videos/mod.rs @@ -0,0 +1,11 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +mod downloading; diff --git a/crates/yt/tests/watch/focus_switch.rs b/crates/yt/tests/watch/focus_switch.rs new file mode 100644 index 0000000..81246f3 --- /dev/null +++ b/crates/yt/tests/watch/focus_switch.rs @@ -0,0 +1,53 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use yt_dlp::json_cast; + +use crate::{_testenv::util, testenv::TestEnv, watch::MpvControl}; + +#[test] +#[ignore = "Currently, this test is missing it's goal"] +fn test_focus_switch() { + let mut env = TestEnv::new(module_path!()); + + { + util::provide_videos(&env, util::Subscription::Tagesschau, 32); + + util::run_select(&env, "s/pick/watch/"); + + env.run(&["download"]); + } + + let mut mpv = MpvControl::new(&mut env); + + assert_pos(&mut mpv, 0); + + for i in 1..32 { + mpv.assert(&["playlist-next", "weak"]); + assert_pos(&mut mpv, i); + } + + mpv.assert(&["playlist-next", "weak"]); + assert_pos(&mut mpv, 2); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 1); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 0); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 0); +} + +fn assert_pos(mpv: &mut MpvControl, pos: i64) { + let mpv_pos = mpv.assert(&["get_property", "playlist-pos"]); + assert_eq!(json_cast!(mpv_pos, as_i64), pos); +} diff --git a/crates/yt/tests/watch/mod.rs b/crates/yt/tests/watch/mod.rs new file mode 100644 index 0000000..7af8b39 --- /dev/null +++ b/crates/yt/tests/watch/mod.rs @@ -0,0 +1,135 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + io::{BufRead, BufReader, Write}, + os::unix::net::UnixStream, + path::PathBuf, + sync::atomic::AtomicU64, +}; + +use colors::{Colorize, IntoCanvas}; +use serde_json::json; +use yt_dlp::{json_cast, json_get, json_try_get}; + +use crate::_testenv::TestEnv; + +mod focus_switch; + +struct MpvControl { + stream: UnixStream, + current_request_id: AtomicU64, + name: &'static str, +} + +impl MpvControl { + fn new(env: &mut TestEnv) -> Self { + let socket_path = { + let stdout = env.run_background(&[ + "watch", + // "--headless", + "--provide-ipc-socket", + ]); + + let line = { + let mut buf = String::new(); + let mut reader = BufReader::new(stdout); + reader.read_line(&mut buf).expect("In-memory"); + buf + }; + + PathBuf::from(line.trim()) + }; + + let stream = UnixStream::connect(&socket_path).unwrap_or_else(|e| { + panic!( + "Path to socket ('{}') should exist, but did not: {e}", + socket_path.display() + ) + }); + + let mut me = Self { + stream, + name: env.name, + current_request_id: AtomicU64::new(0), + }; + + // Disable all events. + // We do not use them, and this should reduce the read load on the socket. + me.assert(&["disable_event", "all"]); + + me + } + + /// Run a command and assert that it ran successfully. + fn assert(&mut self, args: &[&str]) -> serde_json::Value { + let out = self.command(args); + + out.unwrap_or_else(|e| panic!("`mpv {}` failed; error {e}.", args.join(" "))) + } + + /// Run a command in mpv. + /// Will return true if the command ran correctly and false if not. + fn command(&mut self, args: &[&str]) -> Result<serde_json::Value, String> { + let tl_rid = self + .current_request_id + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + eprint!( + "{} `mpv {}`", + self.name.blue().italic().render(true), + args.join(" ") + ); + + writeln!( + self.stream, + "{}", + json!( { "command": args, "request_id": tl_rid }) + ) + .expect("Should always work"); + + loop { + let response: serde_json::Value = { + let mut reader = BufReader::new(&mut self.stream); + + let mut buf = String::new(); + reader.read_line(&mut buf).expect("Works"); + serde_json::from_str(&buf).expect("Mpv only returns json") + }; + + if let Some(rid) = json_try_get!(response, "request_id", as_u64) { + if rid == tl_rid { + let error = json_get!(response, "error", as_str); + + if error == "success" { + let data: serde_json::Value = { + match response.get("data") { + Some(val) => val.to_owned(), + None => serde_json::Value::Null, + } + }; + + eprintln!(", {}: {data}", "output".bright_blue().render(true),); + return Ok(data); + } + + eprintln!(", {}: {error}", "error".bright_red().render(true)); + return Err(error.to_owned()); + } + } + } + } +} + +impl Drop for MpvControl { + fn drop(&mut self) { + self.assert(&["quit"]); + } +} diff --git a/crates/yt_dlp/Cargo.toml b/crates/yt_dlp/Cargo.toml index a948a34..87bb610 100644 --- a/crates/yt_dlp/Cargo.toml +++ b/crates/yt_dlp/Cargo.toml @@ -10,7 +10,7 @@ [package] name = "yt_dlp" -description = "A wrapper around the python yt_dlp library" +description = "A rust ffi wrapper library for the python yt_dlp library" keywords = [] categories = [] version.workspace = true @@ -19,19 +19,18 @@ authors.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true -publish = false +publish = true [dependencies] -pyo3 = { version = "0.23.4", features = ["auto-initialize"] } -bytes.workspace = true +curl = "0.4.48" log.workspace = true -serde.workspace = true +pyo3 = { workspace = true } +pyo3-pylogger = { path = "crates/pyo3-pylogger" } +serde = { workspace = true, features = ["derive"] } serde_json.workspace = true +thiserror = "2.0.12" url.workspace = true -[dev-dependencies] -tokio.workspace = true - [lints] workspace = true diff --git a/crates/yt_dlp/README.md b/crates/yt_dlp/README.md index 591ef2e..ece8540 100644 --- a/crates/yt_dlp/README.md +++ b/crates/yt_dlp/README.md @@ -12,7 +12,7 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. # Yt_py -> \[can be empty\] +> [can be empty] Some text about the project. diff --git a/crates/yt_dlp/.cargo/config.toml b/crates/yt_dlp/crates/pyo3-pylogger/.gitignore index d84f14d..733c5bc 100644 --- a/crates/yt_dlp/.cargo/config.toml +++ b/crates/yt_dlp/crates/pyo3-pylogger/.gitignore @@ -1,12 +1,13 @@ # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 # # This file is part of Yt. # # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -[env] -PYO3_PYTHON = "/nix/store/7xzk119acyws2c4ysygdv66l0grxkr39-python3-3.11.9-env/bin/python3" +target +Cargo.lock +.idea diff --git a/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml b/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml new file mode 100644 index 0000000..28dfacd --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml @@ -0,0 +1,31 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +[package] +name = "pyo3-pylogger" +version = "1.8.0" +edition = "2021" +authors = [ + "Dylan Bobby Storey <dylan.storey@gmail.com>", + "cpu <daniel@binaryparadox.net>", + "Warren Snipes <contact@warrensnipes.dev>", +] +description = "Enables `log` for pyo3 based Rust applications using the `logging` modules." +publish = ["crates-io"] +license = "Apache-2.0" +readme = "README.md" +homepage = "https://github.com/dylanbstorey/pyo3-pylogger" +repository = "https://github.com/dylanbstorey/pyo3-pylogger" +documentation = "https://github.com/dylanbstorey/pyo3-pylogger" + +[dependencies] +pyo3 = { workspace = true } +log = { workspace = true } +phf = { version = "0.12", features = ["macros"] } diff --git a/crates/yt_dlp/crates/pyo3-pylogger/LICENSE b/crates/yt_dlp/crates/pyo3-pylogger/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/yt_dlp/crates/pyo3-pylogger/README.md b/crates/yt_dlp/crates/pyo3-pylogger/README.md new file mode 100644 index 0000000..e68903b --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/README.md @@ -0,0 +1,160 @@ +<!-- +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +SPDX-License-Identifier: Apache-2.0 + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. +--> + +# pyo3-pylogger + +Enables log messages for pyo3 embedded Python applications using Python's +`logging` or module. + +# Features + +- Logging integration between Python's `logging` module and Rust's `log` crate +- Structured logging support via the logging + [extra](https://docs.python.org/3/library/logging.html#logging.Logger.debug) + field (requires `kv` or `tracing-kv`feature) +- Integration with Rust's `tracing` library (requires `tracing` feature) + +# Usage + +```rust +use log::{info, warn}; +use pyo3::{ffi::c_str, prelude::*}; +fn main() { + // register the host handler with python logger, providing a logger target + pyo3_pylogger::register("example_application_py_logger"); + + // initialize up a logger + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace")).init(); + //just show the logger working from Rust. + info!("Just some normal information!"); + warn!("Something spooky happened!"); + + // Ask pyo3 to set up embedded Python interpreter + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + // Python code can now `import logging` as usual + py.run( + c_str!( + r#" +import logging +logging.getLogger().setLevel(0) +logging.debug('DEBUG') +logging.info('INFO') +logging.warning('WARNING') +logging.error('ERROR') +logging.getLogger('foo.bar.baz').info('INFO')"# + ), + None, + None, + ) + .unwrap(); + }) +} + + +``` + +## Outputs + +```bash +[2025-03-28T01:12:29Z INFO helloworld] Just some normal information! +[2025-03-28T01:12:29Z WARN helloworld] Something spooky happened! +[2025-03-28T01:12:29Z DEBUG example_application_py_logger] DEBUG +[2025-03-28T01:12:29Z INFO example_application_py_logger] INFO +[2025-03-28T01:12:29Z WARN example_application_py_logger] WARNING +[2025-03-28T01:12:29Z ERROR example_application_py_logger] ERROR +[2025-03-28T01:12:29Z INFO example_application_py_logger::foo::bar::baz] INFO +``` + +## Structured Logging + +To enable structured logging support, add the `kv` feature to your `Cargo.toml`: + +```toml +[dependencies] +pyo3-pylogger = { version = "0.4", features = ["kv"] } +``` + +Then you can use Python's `extra` parameter to pass structured data: + +```python +logging.info("Processing order", extra={"order_id": "12345", "amount": 99.99}) +``` + +When using a structured logging subscriber in Rust, these key-value pairs will +be properly captured, for example: + +```bash +[2025-03-28T01:12:29Z INFO example_application_py_logger] Processing order order_id=12345 amount=99.99 +``` + +## Tracing Support + +To enable integration with Rust's `tracing` library, add the `tracing` feature +to your `Cargo.toml`: + +```toml +[dependencies] +pyo3-pylogger = { version = "0.4", default-features = false, features = ["tracing"] } +``` + +When the `tracing` feature is enabled, Python logs will be forwarded to the +active tracing subscriber: + +```rust +use tracing::{info, warn}; +use pyo3::{ffi::c_str, prelude::*}; + +fn main() { + // Register the tracing handler with Python logger + pyo3_pylogger::register_tracing("example_application_py_logger"); + + // Initialize tracing subscriber + tracing_subscriber::fmt::init(); + + // Tracing events from Rust + info!("Tracing information from Rust"); + + // Python logging will be captured by the tracing subscriber + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + py.run( + c_str!( + r#" +import logging +logging.getLogger().setLevel(0) +logging.info('This will be captured by tracing')"# + ), + None, + None, + ) + .unwrap(); + }) +} +``` + +### Structured Data with Tracing + +The `tracing` feature automatically supports Python's `extra` field for +structured data. However, the KV fields are json serialized and not available as +tracing attributes. This is a limitation of the `tracing` library and is not +specific to this crate. See +[this issue](https://github.com/tokio-rs/tracing/issues/372) for more +information. + +# Feature Flags + +- `kv`: Enables structured logging support via Python's `extra` fields. This + adds support for the `log` crate's key-value system. +- `tracing`: Enables integration with Rust's `tracing` library. +- `tracing-kv`: Enables structured logging support via Python's `extra` fields + and integration with Rust's `tracing` library. diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs new file mode 100644 index 0000000..67a0c3e --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs @@ -0,0 +1,127 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! Key-Value handling module for Python LogRecord attributes. +//! +//! This module provides functionality to extract and handle custom key-value pairs +//! from Python LogRecord objects, facilitating integration between Python's logging +//! system and Rust's log crate. + +use pyo3::{ + Bound, PyAny, PyResult, + types::{PyAnyMethods, PyDict, PyDictMethods, PyListMethods}, +}; +use std::collections::HashMap; + +/// A static hashset containing all standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes defined in the CPython logging module. +/// +/// This set is used to differentiate between standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes and custom key-value pairs +/// that users might add to their log records. The attributes listed here correspond to the default +/// attributes created by Python's [makeRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L1633-L1634) function. +pub static LOG_RECORD_KV_ATTRIBUTES: phf::Set<&'static str> = phf::phf_set! { + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "taskName", +}; + +/// Extracts custom key-value pairs from a Python LogRecord object. +/// +/// This function examines the `__dict__` of a LogRecord(https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) object and identifies any attributes +/// that are not part of the standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes. These custom attributes are +/// treated as key-value pairs for structured logging. +/// +/// # Arguments +/// * `record` - A reference to a Python LogRecord object +/// +/// # Returns +/// * `PyResult<Option<HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>>>` - If custom attributes +/// are found, returns a HashMap containing the key-value pairs. Returns None if no custom +/// attributes are present. +/// +/// # Note +/// This function relies on the fact that Python will not implement new attributes on the LogRecord object. +/// If new attributes are added, this function will not be able to filter them out and will return them as key-value pairs. +/// In that future, [LOG_RECORD_KV_ATTRIBUTES] will need to be updated to include the new attributes. +/// This is an unfortunate side effect of using the `__dict__` attribute to extract key-value pairs. However, there are no other ways to handle this given that CPython does not distinguish between user-provided attributes and attributes created by the logging module. +pub fn find_kv_args<'a>( + record: &Bound<'a, PyAny>, +) -> PyResult<Option<std::collections::HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>>> { + let dict: Bound<'_, PyDict> = record.getattr("__dict__")?.extract()?; + + // We can abuse the fact that Python dictionaries are ordered by insertion order to reverse iterate over the keys + // and stop at the first key that is not a predefined key-value pair attribute. + let mut kv_args: Option<HashMap<String, pyo3::Bound<'_, pyo3::PyAny>>> = None; + + for item in dict.items().iter().rev() { + let (key, value) = + item.extract::<(pyo3::Bound<'_, pyo3::PyAny>, pyo3::Bound<'_, pyo3::PyAny>)>()?; + + let key_str = key.to_string(); + if LOG_RECORD_KV_ATTRIBUTES.contains(&key_str) { + break; + } + if kv_args.is_none() { + kv_args = Some(HashMap::new()); + } + + kv_args.as_mut().unwrap().insert(key_str, value); + } + + Ok(kv_args) +} + +/// A wrapper struct that implements the `log::kv::Source` trait for Python key-value pairs. +/// +/// This struct allows Python LogRecord custom attributes to be used with Rust's +/// structured logging system by implementing the necessary trait for key-value handling. +/// +/// # Type Parameters +/// * `'a` - The lifetime of the contained Python values +pub struct KVSource<'a>(pub HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>); + +impl log::kv::Source for KVSource<'_> { + /// Visits each key-value pair in the source, converting Python values to debug representations. + /// + /// # Arguments + /// * `visitor` - The visitor that will process each key-value pair + /// + /// # Returns + /// * `Result<(), log::kv::Error>` - Success if all pairs are visited successfully, + /// or an error if visitation fails + fn visit<'kvs>( + &'kvs self, + visitor: &mut dyn log::kv::VisitSource<'kvs>, + ) -> Result<(), log::kv::Error> { + for (key, value) in &self.0 { + let v: log::kv::Value<'_> = log::kv::Value::from_debug(value); + + visitor.visit_pair(log::kv::Key::from_str(key), v)?; + } + Ok(()) + } +} diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs new file mode 100644 index 0000000..d244ef4 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs @@ -0,0 +1,43 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +/// A wrapper type for logging levels that supports both `tracing` and `log` features. +pub(crate) struct Level(pub log::Level); + +/// Converts a numeric level value to the appropriate logging Level. +/// +/// # Arguments +/// +/// * `level` - A u8 value representing the logging level: +/// * 40+ = Error +/// * 30-39 = Warn +/// * 20-29 = Info +/// * 10-19 = Debug +/// * 0-9 = Trace +/// +/// # Returns +/// +/// Returns a `Level` wrapper containing either a `tracing::Level` or `log::Level` +/// depending on which feature is enabled. +pub(crate) fn get_level(level: u8) -> Level { + { + if level.ge(&40u8) { + Level(log::Level::Error) + } else if level.ge(&30u8) { + Level(log::Level::Warn) + } else if level.ge(&20u8) { + Level(log::Level::Info) + } else if level.ge(&10u8) { + Level(log::Level::Debug) + } else { + Level(log::Level::Trace) + } + } +} diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs new file mode 100644 index 0000000..3ecb123 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs @@ -0,0 +1,211 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + ffi::CString, + sync::{self, OnceLock}, +}; + +use log::{debug, log_enabled}; +use pyo3::{ + Bound, Py, PyAny, PyResult, Python, pyfunction, + sync::OnceLockExt, + types::{PyAnyMethods, PyDict, PyListMethods, PyModuleMethods}, + wrap_pyfunction, +}; + +mod kv; +mod level; + +static LOGGER: sync::OnceLock<Py<PyAny>> = OnceLock::new(); + +/// Is the specified record to be logged? Returns false for no, +/// true for yes. Filters can either modify log records in-place or +/// return a completely different record instance which will replace +/// the original log record in any future processing of the event. +#[pyfunction] +fn filter_error_log<'py>(record: Bound<'py, PyAny>) -> bool { + // Filter out all error logs (they are propagated as rust errors) + let levelname: String = record + .getattr("levelname") + .expect("This should exist") + .extract() + .expect("This should be a String"); + + let return_value = levelname.as_str() != "ERROR"; + + if log_enabled!(log::Level::Debug) && !return_value { + let message: String = { + let get_message = record.getattr("getMessage").expect("Is set"); + let message: String = get_message + .call((), None) + .expect("Can be called") + .extract() + .expect("Downcasting works"); + + message.as_str().to_owned() + }; + + debug!("Swollowed error message: '{message}'"); + } + return_value +} + +/// Consume a Python `logging.LogRecord` and emit a Rust `Log` instead. +#[pyfunction] +fn host_log(record: Bound<'_, PyAny>, rust_target: &str) -> PyResult<()> { + let level = record.getattr("levelno")?.extract()?; + let message = record.getattr("getMessage")?.call0()?.to_string(); + let pathname = record.getattr("pathname")?.extract::<String>()?; + let lineno = record.getattr("lineno")?.extract::<u32>()?; + + let logger_name = record.getattr("name")?.extract::<String>()?; + + let full_target: Option<String> = if logger_name.trim().is_empty() || logger_name == "root" { + None + } else { + // Libraries (ex: tracing_subscriber::filter::Directive) expect rust-style targets like foo::bar, + // and may not deal well with "." as a module separator: + let logger_name = logger_name.replace('.', "::"); + Some(format!("{rust_target}::{logger_name}")) + }; + let target = full_target.as_deref().unwrap_or(rust_target); + + handle_record(record, target, &message, lineno, &pathname, level)?; + + Ok(()) +} + +fn handle_record( + #[allow(unused_variables)] record: Bound<'_, PyAny>, + target: &str, + message: &str, + lineno: u32, + pathname: &str, + level: u8, +) -> PyResult<()> { + // If log feature is enabled, use log::logger + let level = crate::level::get_level(level).0; + + { + let mut metadata_builder = log::MetadataBuilder::new(); + metadata_builder.target(target); + metadata_builder.level(level); + + let mut record_builder = log::Record::builder(); + + { + let kv_args = kv::find_kv_args(&record)?; + + let kv_source = kv_args.map(kv::KVSource); + if let Some(kv_source) = kv_source { + log::logger().log( + &record_builder + .metadata(metadata_builder.build()) + .args(format_args!("{}", &message)) + .line(Some(lineno)) + .file(Some(pathname)) + .module_path(Some(pathname)) + .key_values(&kv_source) + .build(), + ); + return Ok(()); + } + } + + log::logger().log( + &record_builder + .metadata(metadata_builder.build()) + .args(format_args!("{}", &message)) + .line(Some(lineno)) + .file(Some(pathname)) + .module_path(Some(pathname)) + .build(), + ); + } + + Ok(()) +} + +/// Registers the host_log function in rust as the event handler for Python's logging logger +/// This function needs to be called from within a pyo3 context as early as possible to ensure logging messages +/// arrive to the rust consumer. +pub fn setup_logging<'py>(py: Python<'py>, target: &str) -> PyResult<Bound<'py, PyAny>> { + let logger = LOGGER + .get_or_init_py_attached(py, || match setup_logging_inner(py, target) { + Ok(ok) => ok.unbind(), + Err(err) => { + panic!("Failed to initialize logger: {}", err); + } + }) + .clone_ref(py); + + Ok(logger.into_bound(py)) +} + +fn setup_logging_inner<'py>(py: Python<'py>, target: &str) -> PyResult<Bound<'py, PyAny>> { + let logging = py.import("logging")?; + + logging.setattr("host_log", wrap_pyfunction!(host_log, &logging)?)?; + + #[allow(clippy::uninlined_format_args)] + let code = CString::new(format!( + r#" +class HostHandler(Handler): + def __init__(self, level=0): + super().__init__(level=level) + + def emit(self, record: LogRecord): + host_log(record, "{}") + +oldBasicConfig = basicConfig +def basicConfig(*pargs, **kwargs): + if "handlers" not in kwargs: + kwargs["handlers"] = [HostHandler()] + return oldBasicConfig(*pargs, **kwargs) +"#, + target + ))?; + + let logging_scope = logging.dict(); + py.run(&code, Some(&logging_scope), None)?; + + let all = logging.index()?; + all.append("HostHandler")?; + + let logger = { + let get_logger = logging_scope.get_item("getLogger")?; + get_logger.call((target,), None)? + }; + + { + let basic_config = logging_scope.get_item("basicConfig")?; + basic_config.call( + (), + { + let dict = PyDict::new(py); + + // Ensure that all events are logged by setting + // the log level to NOTSET (we filter on rust's side) + dict.set_item("level", 0)?; + + Some(dict) + } + .as_ref(), + )?; + } + + { + let add_filter = logger.getattr("addFilter")?; + add_filter.call((wrap_pyfunction!(filter_error_log, &logging)?,), None)?; + } + + Ok(logger) +} diff --git a/crates/bytes/update.sh b/crates/yt_dlp/crates/pyo3-pylogger/update.sh index c1a0215..dd3e57e 100755 --- a/crates/bytes/update.sh +++ b/crates/yt_dlp/crates/pyo3-pylogger/update.sh @@ -1,9 +1,9 @@ -#!/usr/bin/env sh +#! /usr/bin/env sh # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 # # This file is part of Yt. # @@ -13,3 +13,5 @@ cd "$(dirname "$0")" || exit 1 [ "$1" = "upgrade" ] && cargo upgrade --incompatible cargo update + +# vim: ft=sh diff --git a/crates/yt_dlp/examples/main.rs b/crates/yt_dlp/examples/main.rs new file mode 100644 index 0000000..e924407 --- /dev/null +++ b/crates/yt_dlp/examples/main.rs @@ -0,0 +1,15 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +fn main() { + let yt_dlp = yt_dlp::options::YoutubeDLOptions::new().build().unwrap(); + + dbg!(yt_dlp.version().unwrap()); +} diff --git a/crates/yt_dlp/src/duration.rs b/crates/yt_dlp/src/duration.rs deleted file mode 100644 index 19181a5..0000000 --- a/crates/yt_dlp/src/duration.rs +++ /dev/null @@ -1,78 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// TODO: This file should be de-duplicated with the same file in the 'yt' crate <2024-06-25> - -#[derive(Debug, Clone, Copy)] -pub struct Duration { - time: u32, -} - -impl From<&str> for Duration { - fn from(v: &str) -> Self { - let buf: Vec<_> = v.split(':').take(2).collect(); - Self { - time: (buf[0].parse::<u32>().expect("Should be a number") * 60) - + buf[1].parse::<u32>().expect("Should be a number"), - } - } -} - -impl From<Option<f64>> for Duration { - fn from(value: Option<f64>) -> Self { - Self { - #[allow( - clippy::cast_possible_truncation, - clippy::cast_precision_loss, - clippy::cast_sign_loss - )] - time: value.unwrap_or(0.0).ceil() as u32, - } - } -} - -impl std::fmt::Display for Duration { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - const SECOND: u32 = 1; - const MINUTE: u32 = 60 * SECOND; - const HOUR: u32 = 60 * MINUTE; - - let base_hour = self.time - (self.time % HOUR); - let base_min = (self.time % HOUR) - ((self.time % HOUR) % MINUTE); - let base_sec = (self.time % HOUR) % MINUTE; - - let h = base_hour / HOUR; - let m = base_min / MINUTE; - let s = base_sec / SECOND; - - if self.time == 0 { - write!(f, "0s") - } else if h > 0 { - write!(f, "{h}h {m}m") - } else { - write!(f, "{m}m {s}s") - } - } -} -#[cfg(test)] -mod test { - use super::Duration; - - #[test] - fn test_display_duration_1h() { - let dur = Duration { time: 60 * 60 }; - assert_eq!("1h 0m".to_owned(), dur.to_string()); - } - #[test] - fn test_display_duration_30min() { - let dur = Duration { time: 60 * 30 }; - assert_eq!("30m 0s".to_owned(), dur.to_string()); - } -} diff --git a/crates/yt_dlp/src/error.rs b/crates/yt_dlp/src/error.rs deleted file mode 100644 index 3881f0b..0000000 --- a/crates/yt_dlp/src/error.rs +++ /dev/null @@ -1,68 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{fmt::Display, io}; - -use pyo3::Python; - -#[derive(Debug)] -#[allow(clippy::module_name_repetitions)] -pub enum YtDlpError { - ResponseParseError { - error: serde_json::error::Error, - }, - PythonError { - error: Box<pyo3::PyErr>, - kind: String, - }, - IoError { - error: io::Error, - }, -} - -impl std::error::Error for YtDlpError {} - -impl Display for YtDlpError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - YtDlpError::ResponseParseError { error } => write!( - f, - include_str!("./python_json_decode_failed.error_msg"), - error - ), - YtDlpError::PythonError { error, kind: _ } => write!(f, "Python error: {error}"), - YtDlpError::IoError { error } => write!(f, "Io error: {error}"), - } - } -} - -impl From<serde_json::error::Error> for YtDlpError { - fn from(value: serde_json::error::Error) -> Self { - Self::ResponseParseError { error: value } - } -} - -impl From<pyo3::PyErr> for YtDlpError { - fn from(value: pyo3::PyErr) -> Self { - Python::with_gil(|py| { - let kind = value.get_type(py).to_string(); - Self::PythonError { - error: Box::new(value), - kind, - } - }) - } -} - -impl From<io::Error> for YtDlpError { - fn from(value: io::Error) -> Self { - Self::IoError { error: value } - } -} diff --git a/crates/yt_dlp/src/info_json.rs b/crates/yt_dlp/src/info_json.rs new file mode 100644 index 0000000..3ed08ee --- /dev/null +++ b/crates/yt_dlp/src/info_json.rs @@ -0,0 +1,56 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use pyo3::{ + Bound, Python, intern, + types::{PyAnyMethods, PyDict}, +}; + +pub type InfoJson = serde_json::Map<String, serde_json::Value>; + +/// # Panics +/// If expectation about python operations fail. +#[must_use] +pub fn json_loads( + input: serde_json::Map<String, serde_json::Value>, + py: Python<'_>, +) -> Bound<'_, PyDict> { + let json = py.import(intern!(py, "json")).expect("Module exists"); + let loads = json.getattr(intern!(py, "loads")).expect("Method exists"); + let self_str = serde_json::to_string(&serde_json::Value::Object(input)).expect("Vaild json"); + let dict = loads + .call((self_str,), None) + .expect("Vaild json is always a valid dict"); + + dict.downcast_into().expect("Should always be a dict") +} + +/// # Panics +/// If expectation about python operations fail. +#[must_use] +pub fn json_dumps(input: &Bound<'_, PyDict>) -> serde_json::Map<String, serde_json::Value> { + let py = input.py(); + + let json = py.import(intern!(py, "json")).expect("Module exists"); + let dumps = json.getattr(intern!(py, "dumps")).expect("Method exists"); + let dict = dumps + .call((input,), None) + .map_err(|err| err.print(py)) + .expect("Might not always work, but for our dicts it works"); + + let string: String = dict.extract().expect("Should always be a string"); + + let value: serde_json::Value = serde_json::from_str(&string).expect("Should be valid json"); + + match value { + serde_json::Value::Object(map) => map, + _ => unreachable!("These should not be json.dumps output"), + } +} diff --git a/crates/yt_dlp/src/lib.rs b/crates/yt_dlp/src/lib.rs index 40610c2..6be5e87 100644 --- a/crates/yt_dlp/src/lib.rs +++ b/crates/yt_dlp/src/lib.rs @@ -1,6 +1,6 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // // This file is part of Yt. @@ -8,544 +8,371 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -// The pyo3 `pyfunction` proc-macros call unsafe functions internally, which trigger this lint. -#![allow(unsafe_op_in_unsafe_fn)] -#![allow(clippy::missing_errors_doc)] +//! The `yt_dlp` interface is completely contained in the [`YoutubeDL`] structure. -use std::io::stderr; -use std::{env, process}; -use std::{fs::File, io::Write}; +use std::path::PathBuf; -use std::{path::PathBuf, sync::Once}; - -use crate::{duration::Duration, logging::setup_logging, wrapper::info_json::InfoJson}; - -use bytes::Bytes; -use error::YtDlpError; -use log::{Level, debug, info, log_enabled}; -use pyo3::types::{PyString, PyTuple, PyTupleMethods}; +use log::{debug, info}; use pyo3::{ - Bound, PyAny, PyResult, Python, pyfunction, - types::{PyAnyMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyModule}, - wrap_pyfunction, + Bound, Py, PyAny, Python, intern, + types::{PyAnyMethods, PyDict, PyIterator, PyList}, }; -use serde::Serialize; -use serde_json::{Map, Value}; use url::Url; -pub mod duration; -pub mod error; -pub mod logging; -pub mod wrapper; - -#[cfg(test)] -mod tests; - -/// Synchronisation helper, to ensure that we don't setup the logger multiple times -static SYNC_OBJ: Once = Once::new(); - -/// Add a logger to the yt-dlp options. -/// If you have an logger set (i.e. for rust), than this will log to rust -/// -/// # Panics -/// This should never panic. -pub fn add_logger_and_sig_handler<'a>( - opts: Bound<'a, PyDict>, - py: Python<'_>, -) -> PyResult<Bound<'a, PyDict>> { - /// Is the specified record to be logged? Returns false for no, - /// true for yes. Filters can either modify log records in-place or - /// return a completely different record instance which will replace - /// the original log record in any future processing of the event. - #[pyfunction] - fn filter_error_log(_py: Python<'_>, record: &Bound<'_, PyAny>) -> bool { - // Filter out all error logs (they are propagated as rust errors) - let levelname: String = record - .getattr("levelname") - .expect("This should exist") - .extract() - .expect("This should be a String"); - - let return_value = levelname.as_str() != "ERROR"; - - if log_enabled!(Level::Debug) && !return_value { - let message: String = record - .call_method0("getMessage") - .expect("This method exists") - .extract() - .expect("The message is a string"); +use crate::{ + info_json::{InfoJson, json_dumps, json_loads}, + python_error::{IntoPythonError, PythonError}, +}; - debug!("Swollowed error message: '{message}'"); +pub mod info_json; +pub mod options; +pub mod post_processors; +pub mod progress_hook; +pub mod python_error; + +#[macro_export] +macro_rules! json_get { + ($value:expr, $name:literal, $into:ident) => {{ + match $value.get($name) { + Some(val) => $crate::json_cast!(@log_key $name, val, $into), + None => panic!( + concat!( + "Expected '", + $name, + "' to be a key for the '", + stringify!($value), + "' object: {:#?}" + ), + $value + ), } - return_value - } - - setup_logging(py, "yt_dlp")?; - - let logging = PyModule::import(py, "logging")?; - let ytdl_logger = logging.call_method1("getLogger", ("yt_dlp",))?; - - // Ensure that all events are logged by setting the log level to NOTSET (we filter on rust's side) - // Also use this static, to ensure that we don't configure the logger every time - SYNC_OBJ.call_once(|| { - // Disable the SIGINT (Ctrl+C) handler, python installs. - // This allows the user to actually stop the application with Ctrl+C. - // This is here because it can only be run in the main thread and this was here already. - py.run( - c"\ -import signal -signal.signal(signal.SIGINT, signal.SIG_DFL)", - None, - None, - ) - .expect("This code should always work"); - - let config_opts = PyDict::new(py); - config_opts - .set_item("level", 0) - .expect("Setting this item should always work"); - - logging - .call_method("basicConfig", (), Some(&config_opts)) - .expect("This method exists"); - }); - - ytdl_logger.call_method1( - "addFilter", - (wrap_pyfunction!(filter_error_log, py).expect("This function can be wrapped"),), - )?; - - // This was taken from `ytcc`, I don't think it is still applicable - // ytdl_logger.setattr("propagate", false)?; - // let logging_null_handler = logging.call_method0("NullHandler")?; - // ytdl_logger.setattr("addHandler", logging_null_handler)?; - - opts.set_item("logger", ytdl_logger).expect("Should work"); - - Ok(opts) + }}; } -#[pyfunction] -#[allow(clippy::too_many_lines)] -#[allow(clippy::missing_panics_doc)] -#[allow(clippy::items_after_statements)] -#[allow( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::cast_precision_loss -)] -pub fn progress_hook(py: Python<'_>, input: &Bound<'_, PyDict>) -> PyResult<()> { - // Only add the handler, if the log-level is higher than Debug (this avoids covering debug - // messages). - if log_enabled!(Level::Debug) { - return Ok(()); - } - - // ANSI ESCAPE CODES Wrappers {{{ - // see: https://en.wikipedia.org/wiki/ANSI_escape_code#Control_Sequence_Introducer_commands - const CSI: &str = "\x1b["; - fn clear_whole_line() { - eprint!("{CSI}2K"); - } - fn move_to_col(x: usize) { - eprint!("{CSI}{x}G"); - } - // }}} - - let input: Map<String, Value> = serde_json::from_str(&json_dumps( - py, - input - .downcast::<PyAny>() - .expect("Will always work") - .to_owned(), - )?) - .expect("python's json is valid"); - - macro_rules! get { - (@interrogate $item:ident, $type_fun:ident, $get_fun:ident, $name:expr) => {{ - let a = $item.get($name).expect(concat!( - "The field '", - stringify!($name), - "' should exist." - )); - - if a.$type_fun() { - a.$get_fun().expect( - "The should have been checked in the if guard, so unpacking here is fine", - ) +#[macro_export] +macro_rules! json_try_get { + ($value:expr, $name:literal, $into:ident) => {{ + if let Some(val) = $value.get($name) { + if val.is_null() { + None } else { - panic!( - "Value {} => \n{}\n is not of type: {}", - $name, - a, - stringify!($type_fun) - ); + Some(json_cast!(@log_key $name, val, $into)) } - }}; + } else { + None + } + }}; +} - ($type_fun:ident, $get_fun:ident, $name1:expr, $name2:expr) => {{ - let a = get! {@interrogate input, is_object, as_object, $name1}; - let b = get! {@interrogate a, $type_fun, $get_fun, $name2}; - b - }}; +#[macro_export] +macro_rules! json_cast { + ($value:expr, $into:ident) => {{ + let value_name = stringify!($value); + json_cast!(@log_key value_name, $value, $into) + }}; + + (@log_key $name:expr, $value:expr, $into:ident) => {{ + match $value.$into() { + Some(result) => result, + None => panic!( + concat!( + "Expected to be able to cast '{}' value (which is '{:?}') ", + stringify!($into) + ), + $name, + $value + ), + } + }}; +} - ($type_fun:ident, $get_fun:ident, $name:expr) => {{ - get! {@interrogate input, $type_fun, $get_fun, $name} - }}; - } +macro_rules! py_kw_args { + ($py:expr => $($kw_arg_name:ident = $kw_arg_val:expr),*) => {{ + use $crate::python_error::IntoPythonError; - macro_rules! default_get { - (@interrogate $item:ident, $default:expr, $get_fun:ident, $name:expr) => {{ - let a = if let Some(field) = $item.get($name) { - field.$get_fun().unwrap_or($default) - } else { - $default - }; - a - }}; - - ($get_fun:ident, $default:expr, $name1:expr, $name2:expr) => {{ - let a = get! {@interrogate input, is_object, as_object, $name1}; - let b = default_get! {@interrogate a, $default, $get_fun, $name2}; - b - }}; - - ($get_fun:ident, $default:expr, $name:expr) => {{ - default_get! {@interrogate input, $default, $get_fun, $name} - }}; - } + let dict = PyDict::new($py); - macro_rules! c { - ($color:expr, $format:expr) => { - format!("\x1b[{}m{}\x1b[0m", $color, $format) - }; - } + $( + dict.set_item(stringify!($kw_arg_name), $kw_arg_val).wrap_exc($py)?; + )* - fn format_bytes(bytes: u64) -> String { - let bytes = Bytes::new(bytes); - bytes.to_string() + Some(dict) } + .as_ref()}; +} +pub(crate) use py_kw_args; - fn format_speed(speed: f64) -> String { - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - let bytes = Bytes::new(speed.floor() as u64); - format!("{bytes}/s") - } +/// The core of the `yt_dlp` interface. +#[derive(Debug)] +pub struct YoutubeDL { + inner: Py<PyAny>, + options: serde_json::Map<String, serde_json::Value>, +} - let get_title = || -> String { - match get! {is_string, as_str, "info_dict", "ext"} { - "vtt" => { - format!( - "Subtitles ({})", - default_get! {as_str, "<No Subtitle Language>", "info_dict", "name"} - ) - } - "webm" | "mp4" | "mp3" | "m4a" => { - default_get! { as_str, "<No title>", "info_dict", "title"}.to_owned() - } - other => panic!("The extension '{other}' is not yet implemented"), - } - }; - - match get! {is_string, as_str, "status"} { - "downloading" => { - let elapsed = default_get! {as_f64, 0.0f64, "elapsed"}; - let eta = default_get! {as_f64, 0.0, "eta"}; - let speed = default_get! {as_f64, 0.0, "speed"}; - - let downloaded_bytes = get! {is_u64, as_u64, "downloaded_bytes"}; - let (total_bytes, bytes_is_estimate): (u64, &'static str) = { - let total_bytes = default_get!(as_u64, 0, "total_bytes"); - if total_bytes == 0 { - let maybe_estimate = default_get!(as_u64, 0, "total_bytes_estimate"); - - if maybe_estimate == 0 { - // The download speed should be in bytes per second and the eta in seconds. - // Thus multiplying them gets us the raw bytes (which were estimated by `yt_dlp`, from their `info.json`) - let bytes_still_needed = (speed * eta).ceil() as u64; - - (downloaded_bytes + bytes_still_needed, "~") - } else { - (maybe_estimate, "~") - } - } else { - (total_bytes, "") - } - }; - let percent: f64 = { - if total_bytes == 0 { - 100.0 - } else { - (downloaded_bytes as f64 / total_bytes as f64) * 100.0 - } - }; +impl YoutubeDL { + /// Fetch the underlying `yt_dlp` and `python` version. + /// + /// # Errors + /// If python attribute access fails. + pub fn version(&self) -> Result<(String, String), PythonError> { + Python::with_gil(|py| { + let yt_dlp = py + .import(intern!(py, "yt_dlp")) + .wrap_exc(py)? + .getattr(intern!(py, "version")) + .wrap_exc(py)? + .getattr(intern!(py, "__version__")) + .wrap_exc(py)? + .extract() + .wrap_exc(py)?; - clear_whole_line(); - move_to_col(1); - - eprint!( - "'{}' [{}/{} at {}] -> [{} of {}{} {}] ", - c!("34;1", get_title()), - c!("33;1", Duration::from(Some(elapsed))), - c!("33;1", Duration::from(Some(eta))), - c!("32;1", format_speed(speed)), - c!("31;1", format_bytes(downloaded_bytes)), - c!("31;1", bytes_is_estimate), - c!("31;1", format_bytes(total_bytes)), - c!("36;1", format!("{:.02}%", percent)) - ); - stderr().flush()?; - } - "finished" => { - eprintln!("-> Finished downloading."); - } - "error" => { - // TODO: This should probably return an Err. But I'm not so sure where the error would - // bubble up to (i.e., who would catch it) <2025-01-21> - eprintln!("-> Error while downloading: {}", get_title()); - process::exit(1); - } - other => unreachable!("'{other}' should not be a valid state!"), - }; + let python = py.version(); - Ok(()) -} + Ok((yt_dlp, python.to_owned())) + }) + } -pub fn add_hooks<'a>(opts: Bound<'a, PyDict>, py: Python<'_>) -> PyResult<Bound<'a, PyDict>> { - if let Some(hooks) = opts.get_item("progress_hooks")? { - let hooks = hooks.downcast::<PyList>()?; - hooks.append(wrap_pyfunction!(progress_hook, py)?)?; + /// Download a given list of URLs. + /// Returns the paths they were downloaded to. + /// + /// # Errors + /// If one of the downloads error. + pub fn download(&self, urls: &[Url]) -> Result<Vec<PathBuf>, extract_info::Error> { + let mut out_paths = Vec::with_capacity(urls.len()); + + for url in urls { + info!("Started downloading url: '{url}'"); + let info_json = self.extract_info(url, true, true)?; + + // Try to work around yt-dlp type weirdness + let result_string = if let Some(filename) = json_try_get!(info_json, "filename", as_str) + { + PathBuf::from(filename) + } else { + PathBuf::from(json_get!( + json_cast!( + json_get!(info_json, "requested_downloads", as_array)[0], + as_object + ), + "filename", + as_str + )) + }; - opts.set_item("progress_hooks", hooks)?; - } else { - // No hooks are set yet - let hooks_list = PyList::new(py, &[wrap_pyfunction!(progress_hook, py)?])?; + out_paths.push(result_string); + info!("Finished downloading url"); + } - opts.set_item("progress_hooks", hooks_list)?; + Ok(out_paths) } - Ok(opts) -} + /// `extract_info(self, url, download=True, ie_key=None, extra_info=None, process=True, force_generic_extractor=False)` + /// + /// Extract and return the information dictionary of the URL + /// + /// Arguments: + /// - `url` URL to extract + /// + /// Keyword arguments: + /// :`download` Whether to download videos + /// :`process` Whether to resolve all unresolved references (URLs, playlist items). + /// Must be True for download to work + /// + /// # Panics + /// If expectations about python fail to hold. + /// + /// # Errors + /// If python operations fail. + pub fn extract_info( + &self, + url: &Url, + download: bool, + process: bool, + ) -> Result<InfoJson, extract_info::Error> { + Python::with_gil(|py| { + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "extract_info")) + .wrap_exc(py)?; + + let result = inner + .call( + (url.to_string(),), + py_kw_args!(py => download = download, process = process), + ) + .wrap_exc(py)? + .downcast_into::<PyDict>() + .expect("This is a dict"); + + // Resolve the generator object + if let Ok(generator) = result.get_item(intern!(py, "entries")) { + if generator.is_instance_of::<PyList>() { + // already resolved. Do nothing + } else if let Ok(generator) = generator.downcast::<PyIterator>() { + // A python generator object. + let max_backlog = json_try_get!(self.options, "playlistend", as_u64) + .map_or(10000, |playlistend| { + usize::try_from(playlistend).expect("Should work") + }); + + let mut out = vec![]; + for output in generator { + out.push(output.wrap_exc(py)?); + + if out.len() == max_backlog { + break; + } + } -/// Take the result of the ie (may be modified) and resolve all unresolved -/// references (URLs, playlist items). -/// -/// It will also download the videos if 'download'. -/// Returns the resolved `ie_result`. -#[allow(clippy::unused_async)] -#[allow(clippy::missing_panics_doc)] -pub async fn process_ie_result( - yt_dlp_opts: &Map<String, Value>, - ie_result: InfoJson, - download: bool, -) -> Result<InfoJson, YtDlpError> { - Python::with_gil(|py| -> Result<InfoJson, YtDlpError> { - let opts = json_map_to_py_dict(yt_dlp_opts, py)?; - - let instance = get_yt_dlp(py, opts)?; - - let args = { - let ie_result = json_loads_str(py, ie_result)?; - (ie_result,) - }; - - let kwargs = PyDict::new(py); - kwargs.set_item("download", download)?; - - let result = instance - .call_method("process_ie_result", args, Some(&kwargs))? - .downcast_into::<PyDict>() - .expect("This is a dict"); - - let result_str = json_dumps(py, result.into_any())?; - - serde_json::from_str(&result_str).map_err(Into::into) - }) -} + result.set_item(intern!(py, "entries"), out).wrap_exc(py)?; + } else { + // Probably some sort of paged list (`OnDemand` or otherwise) + let max_backlog = json_try_get!(self.options, "playlistend", as_u64) + .map_or(10000, |playlistend| { + usize::try_from(playlistend).expect("Should work") + }); -/// `extract_info(self, url, download=True, ie_key=None, extra_info=None, process=True, force_generic_extractor=False)` -/// -/// Extract and return the information dictionary of the URL -/// -/// Arguments: -/// @param url URL to extract -/// -/// Keyword arguments: -/// @param download Whether to download videos -/// @param process Whether to resolve all unresolved references (URLs, playlist items). -/// Must be True for download to work -/// @param `ie_key` Use only the extractor with this key -/// -/// @param `extra_info` Dictionary containing the extra values to add to the info (For internal use only) -/// @`force_generic_extractor` Force using the generic extractor (Deprecated; use `ie_key`='Generic') -#[allow(clippy::unused_async)] -#[allow(clippy::missing_panics_doc)] -pub async fn extract_info( - yt_dlp_opts: &Map<String, Value>, - url: &Url, - download: bool, - process: bool, -) -> Result<InfoJson, YtDlpError> { - Python::with_gil(|py| -> Result<InfoJson, YtDlpError> { - let opts = json_map_to_py_dict(yt_dlp_opts, py)?; - - let instance = get_yt_dlp(py, opts)?; - let args = (url.as_str(),); - - let kwargs = PyDict::new(py); - kwargs.set_item("download", download)?; - kwargs.set_item("process", process)?; - - let result = instance - .call_method("extract_info", args, Some(&kwargs))? - .downcast_into::<PyDict>() - .expect("This is a dict"); - - // Resolve the generator object - if let Some(generator) = result.get_item("entries")? { - if generator.is_instance_of::<PyList>() { - // already resolved. Do nothing - } else { - let max_backlog = yt_dlp_opts.get("playlistend").map_or(10000, |value| { - usize::try_from(value.as_u64().expect("Works")).expect("Should work") - }); + let next = generator.getattr(intern!(py, "getslice")).wrap_exc(py)?; - let mut out = vec![]; - while let Ok(output) = generator.call_method0("__next__") { - out.push(output); + let output = next + .call((), py_kw_args!(py => start = 0, end = max_backlog)) + .wrap_exc(py)?; - if out.len() == max_backlog { - break; - } + result + .set_item(intern!(py, "entries"), output) + .wrap_exc(py)?; } - result.set_item("entries", out)?; } - } - - let result_str = json_dumps(py, result.into_any())?; - if let Ok(confirm) = env::var("YT_STORE_INFO_JSON") { - if confirm == "yes" { - let mut file = File::create("output.info.json")?; - write!(file, "{result_str}").unwrap(); - } - } + let result = self.prepare_info_json(&result, py)?; - serde_json::from_str(&result_str).map_err(Into::into) - }) -} + Ok(result) + }) + } -/// # Panics -/// Only if python fails to return a valid URL. -pub fn unsmuggle_url(smug_url: &Url) -> PyResult<Url> { - Python::with_gil(|py| { - let utils = get_yt_dlp_utils(py)?; - let url = utils - .call_method1("unsmuggle_url", (smug_url.as_str(),))? - .downcast::<PyTuple>()? - .get_item(0)?; - - let url: Url = url - .downcast::<PyString>()? - .to_string() - .parse() - .expect("Python should be able to return a valid url"); - - Ok(url) - }) -} + /// Take the (potentially modified) result of the information extractor (i.e., + /// [`Self::extract_info`] with `process` and `download` set to false) + /// and resolve all unresolved references (URLs, + /// playlist items). + /// + /// It will also download the videos if 'download' is true. + /// Returns the resolved `ie_result`. + /// + /// # Panics + /// If expectations about python fail to hold. + /// + /// # Errors + /// If python operations fail. + pub fn process_ie_result( + &self, + ie_result: InfoJson, + download: bool, + ) -> Result<InfoJson, process_ie_result::Error> { + Python::with_gil(|py| { + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "process_ie_result")) + .wrap_exc(py)?; + + let result = inner + .call( + (json_loads(ie_result, py),), + py_kw_args!(py => download = download), + ) + .wrap_exc(py)? + .downcast_into::<PyDict>() + .expect("This is a dict"); -/// Download a given list of URLs. -/// Returns the paths they were downloaded to. -/// -/// # Panics -/// Only if `yt_dlp` changes their `info_json` schema. -pub async fn download( - urls: &[Url], - download_options: &Map<String, Value>, -) -> Result<Vec<PathBuf>, YtDlpError> { - let mut out_paths = Vec::with_capacity(urls.len()); - - for url in urls { - info!("Started downloading url: '{}'", url); - let info_json = extract_info(download_options, url, true, true).await?; - - // Try to work around yt-dlp type weirdness - let result_string = if let Some(filename) = info_json.filename { - filename - } else { - info_json.requested_downloads.expect("This must exist")[0] - .filename - .clone() - }; + let result = self.prepare_info_json(&result, py)?; - out_paths.push(result_string); - info!("Finished downloading url: '{}'", url); + Ok(result) + }) } - Ok(out_paths) -} - -fn json_map_to_py_dict<'a>( - map: &Map<String, Value>, - py: Python<'a>, -) -> PyResult<Bound<'a, PyDict>> { - let json_string = serde_json::to_string(&map).expect("This must always work"); + /// Close this [`YoutubeDL`] instance, and stop all currently running downloads. + /// + /// # Errors + /// If python operations fail. + pub fn close(&self) -> Result<(), close::Error> { + Python::with_gil(|py| { + debug!("Closing YoutubeDL."); - let python_dict = json_loads(py, json_string)?; + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "close")) + .wrap_exc(py)?; - Ok(python_dict) -} + inner.call0().wrap_exc(py)?; -fn json_dumps(py: Python<'_>, input: Bound<'_, PyAny>) -> PyResult<String> { - // json.dumps(yt_dlp.sanitize_info(input)) - - let yt_dlp = get_yt_dlp(py, PyDict::new(py))?; - let sanitized_result = yt_dlp.call_method1("sanitize_info", (input,))?; + Ok(()) + }) + } - let json = PyModule::import(py, "json")?; - let dumps = json.getattr("dumps")?; + fn prepare_info_json<'py>( + &self, + info: &Bound<'py, PyDict>, + py: Python<'py>, + ) -> Result<InfoJson, prepare::Error> { + let sanitize = self + .inner + .bind(py) + .getattr(intern!(py, "sanitize_info")) + .wrap_exc(py)?; - let output = dumps.call1((sanitized_result,))?; + let value = sanitize.call((info,), None).wrap_exc(py)?; - let output_str = output.extract::<String>()?; + let result = value.downcast::<PyDict>().expect("This should stay a dict"); - Ok(output_str) + Ok(json_dumps(result)) + } } -fn json_loads_str<T: Serialize>(py: Python<'_>, input: T) -> PyResult<Bound<'_, PyDict>> { - let string = serde_json::to_string(&input).expect("Correct json must be pased"); +#[allow(missing_docs)] +pub mod close { + use crate::python_error::PythonError; - json_loads(py, string) + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + } } +#[allow(missing_docs)] +pub mod process_ie_result { + use crate::{prepare, python_error::PythonError}; -fn json_loads(py: Python<'_>, input: String) -> PyResult<Bound<'_, PyDict>> { - // json.loads(input) - - let json = PyModule::import(py, "json")?; - let dumps = json.getattr("loads")?; + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), - let output = dumps.call1((input,))?; - - Ok(output - .downcast::<PyDict>() - .expect("This should always be a PyDict") - .clone()) + #[error("Failed to prepare the info json")] + InfoJsonPrepare(#[from] prepare::Error), + } } +#[allow(missing_docs)] +pub mod extract_info { + use crate::{prepare, python_error::PythonError}; -fn get_yt_dlp_utils(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> { - let yt_dlp = PyModule::import(py, "yt_dlp")?; - let utils = yt_dlp.getattr("utils")?; + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), - Ok(utils) + #[error("Failed to prepare the info json")] + InfoJsonPrepare(#[from] prepare::Error), + } } -fn get_yt_dlp<'a>(py: Python<'a>, opts: Bound<'a, PyDict>) -> PyResult<Bound<'a, PyAny>> { - // Unconditionally set a logger - let opts = add_logger_and_sig_handler(opts, py)?; - let opts = add_hooks(opts, py)?; - - let yt_dlp = PyModule::import(py, "yt_dlp")?; - let youtube_dl = yt_dlp.call_method1("YoutubeDL", (opts,))?; - - Ok(youtube_dl) +#[allow(missing_docs)] +pub mod prepare { + use crate::python_error::PythonError; + + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + } } diff --git a/crates/yt_dlp/src/logging.rs b/crates/yt_dlp/src/logging.rs deleted file mode 100644 index e731502..0000000 --- a/crates/yt_dlp/src/logging.rs +++ /dev/null @@ -1,133 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// This file is taken from: https://github.com/dylanbstorey/pyo3-pylogger/blob/d89e0d6820ebc4f067647e3b74af59dbc4941dd5/src/lib.rs -// It is licensed under the Apache 2.0 License, copyright up to 2024 by Dylan Storey -// It was modified by Benedikt Peetz 2024 - -// The pyo3 `pyfunction` proc-macros call unsafe functions internally, which trigger this lint. -#![allow(unsafe_op_in_unsafe_fn)] - -use std::ffi::CString; - -use log::{Level, MetadataBuilder, Record, logger}; -use pyo3::{ - Bound, PyAny, PyResult, Python, - prelude::{PyAnyMethods, PyListMethods, PyModuleMethods}, - pyfunction, wrap_pyfunction, -}; - -/// Consume a Python `logging.LogRecord` and emit a Rust `Log` instead. -#[allow(clippy::needless_pass_by_value)] -#[pyfunction] -fn host_log(record: Bound<'_, PyAny>, rust_target: &str) -> PyResult<()> { - let level = record.getattr("levelno")?; - let message = record.getattr("getMessage")?.call0()?.to_string(); - let pathname = record.getattr("pathname")?.to_string(); - let lineno = record - .getattr("lineno")? - .to_string() - .parse::<u32>() - .expect("This should always be a u32"); - - let logger_name = record.getattr("name")?.to_string(); - - let full_target: Option<String> = if logger_name.trim().is_empty() || logger_name == "root" { - None - } else { - // Libraries (ex: tracing_subscriber::filter::Directive) expect rust-style targets like foo::bar, - // and may not deal well with "." as a module separator: - let logger_name = logger_name.replace('.', "::"); - Some(format!("{rust_target}::{logger_name}")) - }; - - let target = full_target.as_deref().unwrap_or(rust_target); - - // error - let error_metadata = if level.ge(40u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Error) - .build() - } else if level.ge(30u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Warn) - .build() - } else if level.ge(20u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Info) - .build() - } else if level.ge(10u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Debug) - .build() - } else { - MetadataBuilder::new() - .target(target) - .level(Level::Trace) - .build() - }; - - logger().log( - &Record::builder() - .metadata(error_metadata) - .args(format_args!("{}", &message)) - .line(Some(lineno)) - .file(None) - .module_path(Some(&pathname)) - .build(), - ); - - Ok(()) -} - -/// Registers the `host_log` function in rust as the event handler for Python's logging logger -/// This function needs to be called from within a pyo3 context as early as possible to ensure logging messages -/// arrive to the rust consumer. -/// -/// # Panics -/// Only if internal assertions fail. -#[allow(clippy::module_name_repetitions)] -pub fn setup_logging(py: Python<'_>, target: &str) -> PyResult<()> { - let logging = py.import("logging")?; - - logging.setattr("host_log", wrap_pyfunction!(host_log, &logging)?)?; - - py.run( - CString::new(format!( - r#" -class HostHandler(Handler): - def __init__(self, level=0): - super().__init__(level=level) - - def emit(self, record): - host_log(record,"{target}") - -oldBasicConfig = basicConfig -def basicConfig(*pargs, **kwargs): - if "handlers" not in kwargs: - kwargs["handlers"] = [HostHandler()] - return oldBasicConfig(*pargs, **kwargs) -"# - )) - .expect("This is hardcoded") - .as_c_str(), - Some(&logging.dict()), - None, - )?; - - let all = logging.index()?; - all.append("HostHandler")?; - - Ok(()) -} diff --git a/crates/yt_dlp/src/options.rs b/crates/yt_dlp/src/options.rs new file mode 100644 index 0000000..ad30301 --- /dev/null +++ b/crates/yt_dlp/src/options.rs @@ -0,0 +1,207 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync; + +use pyo3::{ + Bound, IntoPyObjectExt, PyAny, PyResult, Python, intern, + types::{PyAnyMethods, PyCFunction, PyDict, PyTuple}, +}; +use pyo3_pylogger::setup_logging; + +use crate::{ + YoutubeDL, json_loads, post_processors, py_kw_args, + python_error::{IntoPythonError, PythonError}, +}; + +pub type ProgressHookFunction = fn(py: Python<'_>) -> PyResult<Bound<'_, PyCFunction>>; +pub type PostProcessorFunction = fn(py: Python<'_>) -> PyResult<Bound<'_, PyAny>>; + +/// Options, that are used to customize the download behaviour. +/// +/// In the future, this might get a Builder api. +/// +/// See `help(yt_dlp.YoutubeDL())` from python for a full list of available options. +#[derive(Default, Debug)] +pub struct YoutubeDLOptions { + options: serde_json::Map<String, serde_json::Value>, + progress_hook: Option<ProgressHookFunction>, + post_processors: Vec<PostProcessorFunction>, +} + +impl YoutubeDLOptions { + #[must_use] + pub fn new() -> Self { + let me = Self { + options: serde_json::Map::new(), + progress_hook: None, + post_processors: vec![], + }; + + me.with_post_processor(post_processors::dearrow::process) + } + + #[must_use] + pub fn set(self, key: impl Into<String>, value: impl Into<serde_json::Value>) -> Self { + let mut options = self.options; + options.insert(key.into(), value.into()); + + Self { options, ..self } + } + + #[must_use] + pub fn with_progress_hook(self, progress_hook: ProgressHookFunction) -> Self { + if let Some(_previous_hook) = self.progress_hook { + todo!() + } else { + Self { + progress_hook: Some(progress_hook), + ..self + } + } + } + + #[must_use] + pub fn with_post_processor(mut self, pp: PostProcessorFunction) -> Self { + self.post_processors.push(pp); + self + } + + /// # Errors + /// If the underlying [`YoutubeDL::from_options`] errors. + pub fn build(self) -> Result<YoutubeDL, build::Error> { + YoutubeDL::from_options(self) + } + + #[must_use] + pub fn from_json_options(options: serde_json::Map<String, serde_json::Value>) -> Self { + Self { + options, + ..Self::new() + } + } + + #[must_use] + pub fn get(&self, key: &str) -> Option<&serde_json::Value> { + self.options.get(key) + } +} + +impl YoutubeDL { + /// Construct this instance from options. + /// + /// # Panics + /// If `yt_dlp` changed their interface. + /// + /// # Errors + /// If a python call fails. + #[allow(clippy::too_many_lines)] + pub fn from_options(options: YoutubeDLOptions) -> Result<Self, build::Error> { + pyo3::prepare_freethreaded_python(); + + let output_options = options.options.clone(); + + let yt_dlp_module = Python::with_gil(|py| { + let opts = json_loads(options.options, py); + + { + static CALL_ONCE: sync::Once = sync::Once::new(); + + CALL_ONCE.call_once(|| { + py.run( + c" +import signal +signal.signal(signal.SIGINT, signal.SIG_DFL) + ", + None, + None, + ) + .unwrap_or_else(|err| { + panic!("Failed to disable python signal handling: {err}") + }); + }); + } + + { + // Setup the progress hook + if let Some(ph) = options.progress_hook { + opts.set_item(intern!(py, "progress_hooks"), vec![ph(py).wrap_exc(py)?]) + .wrap_exc(py)?; + } + } + + { + // Unconditionally set a logger. + // Otherwise, yt_dlp will log to stderr. + + let ytdl_logger = setup_logging(py, "yt_dlp").wrap_exc(py)?; + + opts.set_item(intern!(py, "logger"), ytdl_logger) + .wrap_exc(py)?; + } + + let inner = { + let p_params = opts.into_bound_py_any(py).wrap_exc(py)?; + let p_auto_init = true.into_bound_py_any(py).wrap_exc(py)?; + + py.import(intern!(py, "yt_dlp.YoutubeDL")) + .wrap_exc(py)? + .getattr(intern!(py, "YoutubeDL")) + .wrap_exc(py)? + .call1( + PyTuple::new( + py, + [ + p_params.into_bound_py_any(py).wrap_exc(py)?, + p_auto_init.into_bound_py_any(py).wrap_exc(py)?, + ], + ) + .wrap_exc(py)?, + ) + .wrap_exc(py)? + }; + + { + // Setup the post processors + let add_post_processor_fun = inner + .getattr(intern!(py, "add_post_processor")) + .wrap_exc(py)?; + + for pp in options.post_processors { + add_post_processor_fun + .call( + (pp(py).wrap_exc(py)?.into_bound_py_any(py).wrap_exc(py)?,), + // "when" can take any value in yt_dlp.utils.POSTPROCESS_WHEN + py_kw_args!(py => when = "pre_process"), + ) + .wrap_exc(py)?; + } + } + + Ok::<_, PythonError>(inner.unbind()) + })?; + + Ok(Self { + inner: yt_dlp_module, + options: output_options, + }) + } +} + +#[allow(missing_docs)] +pub mod build { + use crate::python_error::PythonError; + + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + } +} diff --git a/crates/yt_dlp/src/post_processors/dearrow.rs b/crates/yt_dlp/src/post_processors/dearrow.rs new file mode 100644 index 0000000..f35f301 --- /dev/null +++ b/crates/yt_dlp/src/post_processors/dearrow.rs @@ -0,0 +1,247 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use curl::easy::Easy; +use log::{error, info, trace, warn}; +use pyo3::{ + Bound, PyAny, PyErr, PyResult, Python, exceptions, intern, pyfunction, + types::{PyAnyMethods, PyDict, PyModule}, + wrap_pyfunction, +}; +use serde::{Deserialize, Serialize}; + +use crate::{ + pydict_cast, pydict_get, + python_error::{IntoPythonError, PythonError}, +}; + +/// # Errors +/// - If the underlying function returns an error. +/// - If python operations fail. +pub fn process(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> { + #[pyfunction] + fn actual_processor(info_json: Bound<'_, PyDict>) -> PyResult<Bound<'_, PyDict>> { + let output = match unwrapped_process(info_json) { + Ok(ok) => ok, + Err(err) => { + return Err(PyErr::new::<exceptions::PyRuntimeError, _>(err.to_string())); + } + }; + Ok(output) + } + + let module = PyModule::new(py, "rust_post_processors")?; + let scope = PyDict::new(py); + scope.set_item( + intern!(py, "actual_processor"), + wrap_pyfunction!(actual_processor, module)?, + )?; + py.run( + c" +import yt_dlp + +class DeArrow(yt_dlp.postprocessor.PostProcessor): + def run(self, info): + info = actual_processor(info) + return [], info + +inst = DeArrow() +", + Some(&scope), + None, + )?; + + Ok(scope.get_item(intern!(py, "inst"))?.downcast_into()?) +} + +/// # Errors +/// If the API access fails. +pub fn unwrapped_process(info: Bound<'_, PyDict>) -> Result<Bound<'_, PyDict>, Error> { + if pydict_get!(info, "extractor_key", String).as_str() != "Youtube" { + return Ok(info); + } + + let mut retry_num = 3; + let mut output: DeArrowApi = { + loop { + let output_bytes = { + let mut dst = Vec::new(); + + let mut easy = Easy::new(); + easy.url( + format!( + "https://sponsor.ajay.app/api/branding?videoID={}", + pydict_get!(info, "id", String) + ) + .as_str(), + )?; + + let mut transfer = easy.transfer(); + transfer.write_function(|data| { + dst.extend_from_slice(data); + Ok(data.len()) + })?; + transfer.perform()?; + drop(transfer); + + dst + }; + + match serde_json::from_slice(&output_bytes) { + Ok(ok) => break ok, + Err(err) => { + if retry_num > 0 { + trace!( + "DeArrow: Api access failed, trying again ({retry_num} retries left)" + ); + retry_num -= 1; + } else { + let err: serde_json::Error = err; + return Err(err.into()); + } + } + } + } + }; + + // We pop the titles, so we need this vector reversed. + output.titles.reverse(); + + let title_len = output.titles.len(); + let mut iterator = output.titles.clone(); + let selected = loop { + let Some(title) = iterator.pop() else { + break false; + }; + + if (title.locked || title.votes < 1) && title_len > 1 { + info!( + "DeArrow: Skipping title {:#?}, as it is not good enough", + title.value + ); + // Skip titles that are not “good” enough. + continue; + } + + update_title(&info, &title.value).wrap_exc(info.py())?; + + break true; + }; + + if !selected && title_len != 0 { + // No title was selected, even though we had some titles. + // Just pick the first one in this case. + update_title(&info, &output.titles[0].value).wrap_exc(info.py())?; + } + + Ok(info) +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + + #[error("Failed to access the DeArrow api: {0}")] + Get(#[from] curl::Error), + + #[error("Failed to deserialize a api json return object: {0}")] + Deserialize(#[from] serde_json::Error), +} + +fn update_title(info: &Bound<'_, PyDict>, new_title: &str) -> PyResult<()> { + let py = info.py(); + + assert!(!info.contains(intern!(py, "original_title"))?); + + if let Ok(old_title) = info.get_item(intern!(py, "title")) { + warn!( + "DeArrow: Updating title from {:#?} to {:#?}", + pydict_cast!(old_title, &str), + new_title + ); + + info.set_item(intern!(py, "original_title"), old_title) + .expect("We checked, it is a new key"); + } else { + warn!("DeArrow: Setting title to {new_title:#?}"); + } + + let cleaned_title = { + // NOTE(@bpeetz): DeArrow uses `>` as a “Don't format the next word” mark. + // They should be removed, if one does not use a auto-formatter. <2025-06-16> + new_title.replace('>', "") + }; + + info.set_item(intern!(py, "title"), cleaned_title) + .expect("This should work?"); + + Ok(()) +} + +#[derive(Serialize, Deserialize)] +/// See: <https://wiki.sponsor.ajay.app/w/API_Docs/DeArrow> +struct DeArrowApi { + titles: Vec<Title>, + thumbnails: Vec<Thumbnail>, + + #[serde(alias = "randomTime")] + random_time: Option<f64>, + + #[serde(alias = "videoDuration")] + video_duration: Option<f64>, + + #[serde(alias = "casualVotes")] + casual_votes: Vec<CasualVote>, +} + +#[derive(Serialize, Deserialize)] +struct CasualVote { + id: String, + count: u32, + title: String, +} + +#[derive(Serialize, Deserialize, Clone)] +struct Title { + /// Note: Titles will sometimes contain > before a word. + /// This tells the auto-formatter to not format a word. + /// If you have no auto-formatter, you can ignore this and replace it with an empty string + #[serde(alias = "title")] + value: String, + + original: bool, + votes: u64, + locked: bool, + + #[serde(alias = "UUID")] + uuid: String, + + /// only present if requested + #[serde(alias = "userID")] + user_id: Option<String>, +} + +#[derive(Serialize, Deserialize)] +struct Thumbnail { + // null if original is true + timestamp: Option<f64>, + + original: bool, + votes: u64, + locked: bool, + + #[serde(alias = "UUID")] + uuid: String, + + /// only present if requested + #[serde(alias = "userID")] + user_id: Option<String>, +} diff --git a/crates/yt_dlp/src/post_processors/mod.rs b/crates/yt_dlp/src/post_processors/mod.rs new file mode 100644 index 0000000..d9be3f5 --- /dev/null +++ b/crates/yt_dlp/src/post_processors/mod.rs @@ -0,0 +1,48 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub mod dearrow; + +#[macro_export] +macro_rules! pydict_get { + ($value:expr, $name:literal, $into:ty) => {{ + let item = $value.get_item(pyo3::intern!($value.py(), $name)); + match &item { + Ok(val) => $crate::pydict_cast!(val, $into), + Err(_) => panic!( + concat!( + "Expected '", + $name, + "' to be a key for the'", + stringify!($value), + "' py dictionary: {:#?}" + ), + $value + ), + } + }}; +} + +#[macro_export] +macro_rules! pydict_cast { + ($value:expr, $into:ty) => {{ + match $value.extract::<$into>() { + Ok(result) => result, + Err(val) => panic!( + concat!( + "Expected to be able to extract ", + stringify!($into), + " from value ({:#?})." + ), + val + ), + } + }}; +} diff --git a/crates/yt_dlp/src/progress_hook.rs b/crates/yt_dlp/src/progress_hook.rs new file mode 100644 index 0000000..7e5f8a5 --- /dev/null +++ b/crates/yt_dlp/src/progress_hook.rs @@ -0,0 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +#[macro_export] +macro_rules! wrap_progress_hook { + ($name:ident, $new_name:ident) => { + pub(crate) fn $new_name( + py: yt_dlp::progress_hook::__priv::pyo3::Python<'_>, + ) -> yt_dlp::progress_hook::__priv::pyo3::PyResult< + yt_dlp::progress_hook::__priv::pyo3::Bound< + '_, + yt_dlp::progress_hook::__priv::pyo3::types::PyCFunction, + >, + > { + #[yt_dlp::progress_hook::__priv::pyo3::pyfunction] + #[pyo3(crate = "yt_dlp::progress_hook::__priv::pyo3")] + fn inner( + input: yt_dlp::progress_hook::__priv::pyo3::Bound< + '_, + yt_dlp::progress_hook::__priv::pyo3::types::PyDict, + >, + ) -> yt_dlp::progress_hook::__priv::pyo3::PyResult<()> { + let processed_input = { + let new_dict = yt_dlp::progress_hook::__priv::pyo3::types::PyDict::new(input.py()); + + input + .into_iter() + .filter_map(|(name, value)| { + let real_name = yt_dlp::progress_hook::__priv::pyo3::types::PyAnyMethods::extract::<String>(&name).expect("Should always be a string"); + + if real_name.starts_with('_') { + None + } else { + Some((real_name, value)) + } + }) + .for_each(|(key, value)| { + yt_dlp::progress_hook::__priv::pyo3::types::PyDictMethods::set_item(&new_dict, &key, value) + .expect("This is a transpositions, should always be valid"); + }); + yt_dlp::progress_hook::__priv::json_dumps(&new_dict) + }; + + $name(processed_input)?; + + Ok(()) + } + + let module = yt_dlp::progress_hook::__priv::pyo3::types::PyModule::new(py, "progress_hook")?; + let fun = yt_dlp::progress_hook::__priv::pyo3::wrap_pyfunction!(inner, module)?; + + Ok(fun) + } + }; +} + +pub mod __priv { + pub use crate::info_json::{json_dumps, json_loads}; + pub use pyo3; +} diff --git a/crates/yt_dlp/src/python_error.rs b/crates/yt_dlp/src/python_error.rs new file mode 100644 index 0000000..0c442b3 --- /dev/null +++ b/crates/yt_dlp/src/python_error.rs @@ -0,0 +1,55 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{self, Display}; + +use log::{Level, debug, log_enabled}; +use pyo3::{PyErr, Python, types::PyTracebackMethods}; + +#[derive(thiserror::Error, Debug)] +pub struct PythonError(pub String); + +pub(crate) trait IntoPythonError<T>: Sized { + fn wrap_exc(self, py: Python<'_>) -> Result<T, PythonError>; +} + +impl<T> IntoPythonError<T> for Result<T, PyErr> { + fn wrap_exc(self, py: Python<'_>) -> Result<T, PythonError> { + self.map_err(|exc| PythonError::from_exception(py, &exc)) + } +} + +impl Display for PythonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Python threw an exception: {}", self.0) + } +} + +impl PythonError { + pub(super) fn from_exception(py: Python<'_>, exc: &PyErr) -> Self { + let buffer = process_exception(py, exc); + Self(buffer) + } +} + +pub(super) fn process_exception(py: Python<'_>, err: &PyErr) -> String { + if log_enabled!(Level::Debug) { + let mut output = err.to_string(); + + if let Some(tb) = err.traceback(py) { + output.push('\n'); + output.push_str(&tb.format().unwrap()); + } + + debug!("Python threw an exception: {output}"); + } + + err.to_string() +} diff --git a/crates/yt_dlp/src/python_json_decode_failed.error_msg b/crates/yt_dlp/src/python_json_decode_failed.error_msg deleted file mode 100644 index d10688e..0000000 --- a/crates/yt_dlp/src/python_json_decode_failed.error_msg +++ /dev/null @@ -1,5 +0,0 @@ -Failed to decode yt-dlp's response: {} - -This is probably a bug. -Try running the command again with the `YT_STORE_INFO_JSON=yes` environment variable set -and maybe debug it further via `yt check info-json output.info.json`. diff --git a/crates/yt_dlp/src/tests.rs b/crates/yt_dlp/src/tests.rs deleted file mode 100644 index 91b6626..0000000 --- a/crates/yt_dlp/src/tests.rs +++ /dev/null @@ -1,89 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::sync::LazyLock; - -use serde_json::{Value, json}; -use url::Url; - -static YT_OPTS: LazyLock<serde_json::Map<String, Value>> = LazyLock::new(|| { - match json!({ - "playliststart": 1, - "playlistend": 10, - "noplaylist": false, - "extract_flat": false, - }) { - Value::Object(obj) => obj, - _ => unreachable!("This json is hardcoded"), - } -}); - -#[tokio::test] -#[ignore = "This test hangs forever"] -async fn test_extract_info_video() { - let info = crate::extract_info( - &YT_OPTS, - &Url::parse("https://www.youtube.com/watch?v=dbjPnXaacAU").expect("Is valid."), - false, - false, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{info:#?}"); -} - -#[tokio::test] -#[ignore = "This test hangs forever"] -async fn test_extract_info_url() { - let err = crate::extract_info( - &YT_OPTS, - &Url::parse("https://google.com").expect("Is valid."), - false, - false, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{err:#?}"); -} - -#[tokio::test] -#[ignore = "This test hangs forever"] -async fn test_extract_info_playlist() { - let err = crate::extract_info( - &YT_OPTS, - &Url::parse("https://www.youtube.com/@TheGarriFrischer/videos").expect("Is valid."), - false, - true, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{err:#?}"); -} -#[tokio::test] -#[ignore = "This test hangs forever"] -async fn test_extract_info_playlist_full() { - let err = crate::extract_info( - &YT_OPTS, - &Url::parse("https://www.youtube.com/@NixOS-Foundation/videos").expect("Is valid."), - false, - true, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{err:#?}"); -} diff --git a/crates/yt_dlp/src/wrapper/info_json.rs b/crates/yt_dlp/src/wrapper/info_json.rs deleted file mode 100644 index a2c00df..0000000 --- a/crates/yt_dlp/src/wrapper/info_json.rs +++ /dev/null @@ -1,824 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// `yt_dlp` named them like this. -#![allow(clippy::pub_underscore_fields)] - -use std::{collections::HashMap, path::PathBuf}; - -use pyo3::{Bound, PyResult, Python, types::PyDict}; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; -use url::Url; - -use crate::json_loads_str; - -type Todo = String; -type Extractor = String; -type ExtractorKey = String; - -// TODO: Change this to map `_type` to a structure of values, instead of the options <2024-05-27> -// And replace all the strings with better types (enums or urls) -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct InfoJson { - #[serde(skip_serializing_if = "Option::is_none")] - pub __files_to_move: Option<FilesToMove>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub __last_playlist_index: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub __post_extractor: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub __x_forwarded_for_ip: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub _filename: Option<PathBuf>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub _format_sort_fields: Option<Vec<String>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub _has_drm: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub _type: Option<InfoType>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub _version: Option<Version>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub abr: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub acodec: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub age_limit: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub artists: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub aspect_ratio: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub asr: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub audio_channels: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub audio_ext: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub automatic_captions: Option<HashMap<String, Vec<Caption>>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub availability: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub average_rating: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub categories: Option<Vec<String>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub channel: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub channel_follower_count: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub channel_id: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub channel_is_verified: Option<bool>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub channel_url: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub chapters: Option<Vec<Chapter>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub comment_count: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub comments: Option<Vec<Comment>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_view_count: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub container: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub direct: Option<bool>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub display_id: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub downloader_options: Option<DownloaderOptions>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub duration: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub duration_string: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub dynamic_range: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub entries: Option<Vec<InfoJson>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub episode: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub episode_number: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub epoch: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub ext: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub extractor: Option<Extractor>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub extractor_key: Option<ExtractorKey>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub filename: Option<PathBuf>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub filesize: Option<u64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub filesize_approx: Option<u64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub format: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub format_id: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub format_index: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub format_note: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub formats: Option<Vec<Format>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub fps: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub fulltitle: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub genre: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub genres: Option<Vec<String>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub has_drm: Option<bool>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub heatmap: Option<Vec<HeatMapEntry>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub height: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub hls_aes: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub http_headers: Option<HttpHeader>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub id: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub ie_key: Option<ExtractorKey>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub is_live: Option<bool>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub language: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub language_preference: Option<i32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub license: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub like_count: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub live_status: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub location: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub manifest_url: Option<Url>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub media_type: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub modified_date: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub n_entries: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub original_url: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playable_in_embed: Option<bool>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_autonumber: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_channel: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_channel_id: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_count: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_id: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_index: Option<u64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_title: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_uploader: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_uploader_id: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub playlist_webpage_url: Option<Url>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub preference: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub protocol: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub quality: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub release_date: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub release_timestamp: Option<u64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub release_year: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub repost_count: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub requested_downloads: Option<Vec<RequestedDownloads>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub requested_entries: Option<Vec<u32>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub requested_formats: Option<Vec<Format>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub requested_subtitles: Option<HashMap<String, Subtitle>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub resolution: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub season: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub season_number: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub series: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub source_preference: Option<i32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub sponsorblock_chapters: Option<Vec<SponsorblockChapter>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub stretched_ratio: Option<Todo>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub subtitles: Option<HashMap<String, Vec<Caption>>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub tags: Option<Vec<String>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub tbr: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub thumbnail: Option<Url>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub thumbnails: Option<Vec<ThumbNail>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub timestamp: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub title: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub upload_date: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub uploader: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub uploader_id: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub uploader_url: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub url: Option<Url>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub vbr: Option<f64>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub vcodec: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub video_ext: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub view_count: Option<u32>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub was_live: Option<bool>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub webpage_url: Option<Url>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub webpage_url_basename: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub webpage_url_domain: Option<String>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub width: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -#[allow(missing_copy_implementations)] -pub struct FilesToMove {} - -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RequestedDownloads { - pub __files_to_merge: Option<Vec<Todo>>, - pub __finaldir: PathBuf, - pub __infojson_filename: PathBuf, - pub __postprocessors: Vec<Todo>, - pub __real_download: bool, - pub __write_download_archive: bool, - pub _filename: PathBuf, - pub _type: InfoType, - pub _version: Version, - pub abr: f64, - pub acodec: String, - pub aspect_ratio: Option<f64>, - pub asr: Option<u32>, - pub audio_channels: Option<u32>, - pub audio_ext: Option<String>, - pub chapters: Option<Vec<SponsorblockChapter>>, - pub duration: Option<f64>, - pub dynamic_range: Option<String>, - pub ext: String, - pub filename: PathBuf, - pub filepath: PathBuf, - pub filesize_approx: Option<u64>, - pub format: String, - pub format_id: String, - pub format_note: Option<String>, - pub fps: Option<f64>, - pub has_drm: Option<bool>, - pub height: Option<u32>, - pub http_headers: Option<HttpHeader>, - pub infojson_filename: PathBuf, - pub language: Option<String>, - pub manifest_url: Option<Url>, - pub protocol: String, - pub quality: Option<i64>, - pub requested_formats: Option<Vec<Format>>, - pub resolution: String, - pub tbr: f64, - pub url: Option<Url>, - pub vbr: f64, - pub vcodec: String, - pub video_ext: Option<String>, - pub width: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Subtitle { - pub ext: SubtitleExt, - pub filepath: PathBuf, - pub filesize: Option<u64>, - pub fragment_base_url: Option<Url>, - pub fragments: Option<Vec<Fragment>>, - pub manifest_url: Option<Url>, - pub name: Option<String>, - pub protocol: Option<Todo>, - pub url: Url, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -pub enum SubtitleExt { - #[serde(alias = "vtt")] - Vtt, - - #[serde(alias = "mp4")] - Mp4, - - #[serde(alias = "json")] - Json, - #[serde(alias = "json3")] - Json3, - - #[serde(alias = "ttml")] - Ttml, - - #[serde(alias = "srv1")] - Srv1, - #[serde(alias = "srv2")] - Srv2, - #[serde(alias = "srv3")] - Srv3, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Caption { - pub ext: SubtitleExt, - pub filepath: Option<PathBuf>, - pub filesize: Option<u64>, - pub fragments: Option<Vec<SubtitleFragment>>, - pub fragment_base_url: Option<Url>, - pub manifest_url: Option<Url>, - pub name: Option<String>, - pub protocol: Option<String>, - pub url: String, - pub video_id: Option<String>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct SubtitleFragment { - path: PathBuf, - duration: Option<f64>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Chapter { - pub end_time: f64, - pub start_time: f64, - pub title: String, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct SponsorblockChapter { - /// This is an utterly useless field, and should thus be ignored - pub _categories: Option<Vec<Vec<Value>>>, - - pub categories: Option<Vec<SponsorblockChapterCategory>>, - pub category: Option<SponsorblockChapterCategory>, - pub category_names: Option<Vec<String>>, - pub end_time: f64, - pub name: Option<String>, - pub r#type: Option<SponsorblockChapterType>, - pub start_time: f64, - pub title: String, -} - -pub fn get_none<'de, D, T>(_: D) -> Result<Option<T>, D::Error> -where - D: Deserializer<'de>, -{ - Ok(None) -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub enum SponsorblockChapterType { - #[serde(alias = "skip")] - Skip, - - #[serde(alias = "chapter")] - Chapter, - - #[serde(alias = "poi")] - Poi, -} -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub enum SponsorblockChapterCategory { - #[serde(alias = "filler")] - Filler, - - #[serde(alias = "interaction")] - Interaction, - - #[serde(alias = "music_offtopic")] - MusicOfftopic, - - #[serde(alias = "poi_highlight")] - PoiHighlight, - - #[serde(alias = "preview")] - Preview, - - #[serde(alias = "sponsor")] - Sponsor, - - #[serde(alias = "selfpromo")] - SelfPromo, - - #[serde(alias = "chapter")] - Chapter, - - #[serde(alias = "intro")] - Intro, - - #[serde(alias = "outro")] - Outro, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -#[allow(missing_copy_implementations)] -pub struct HeatMapEntry { - pub start_time: f64, - pub end_time: f64, - pub value: f64, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub enum InfoType { - #[serde(alias = "playlist")] - #[serde(rename(serialize = "playlist"))] - Playlist, - - #[serde(alias = "url")] - #[serde(rename(serialize = "url"))] - Url, - - #[serde(alias = "video")] - #[serde(rename(serialize = "video"))] - Video, -} - -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct Version { - pub current_git_head: Option<String>, - pub release_git_head: String, - pub repository: String, - pub version: String, -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(from = "String")] -#[serde(deny_unknown_fields)] -pub enum Parent { - Root, - Id(String), -} - -impl Parent { - #[must_use] - pub fn id(&self) -> Option<&str> { - if let Self::Id(id) = self { - Some(id) - } else { - None - } - } -} - -impl From<String> for Parent { - fn from(value: String) -> Self { - if value == "root" { - Self::Root - } else { - Self::Id(value) - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(from = "String")] -#[serde(deny_unknown_fields)] -pub struct Id { - pub id: String, -} -impl From<String> for Id { - fn from(value: String) -> Self { - Self { - // Take the last element if the string is split with dots, otherwise take the full id - id: value.split('.').last().unwrap_or(&value).to_owned(), - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -#[allow(clippy::struct_excessive_bools)] -pub struct Comment { - pub id: Id, - pub text: String, - #[serde(default = "zero")] - pub like_count: u32, - pub is_pinned: bool, - pub author_id: String, - #[serde(default = "unknown")] - pub author: String, - pub author_is_verified: bool, - pub author_thumbnail: Url, - pub parent: Parent, - #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] - pub edited: bool, - // Can't also be deserialized, as it's already used in 'edited' - // _time_text: String, - pub timestamp: i64, - pub author_url: Option<Url>, - pub author_is_uploader: bool, - pub is_favorited: bool, -} -fn unknown() -> String { - "<Unknown>".to_string() -} -fn zero() -> u32 { - 0 -} -fn edited_from_time_text<'de, D>(d: D) -> Result<bool, D::Error> -where - D: Deserializer<'de>, -{ - let s = String::deserialize(d)?; - if s.contains(" (edited)") { - Ok(true) - } else { - Ok(false) - } -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct ThumbNail { - pub id: Option<String>, - pub preference: Option<i32>, - /// in the form of "[`height`]x[`width`]" - pub resolution: Option<String>, - pub url: Url, - pub width: Option<u32>, - pub height: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Format { - pub __needs_testing: Option<bool>, - pub __working: Option<bool>, - pub abr: Option<f64>, - pub acodec: Option<String>, - pub aspect_ratio: Option<f64>, - pub asr: Option<f64>, - pub audio_channels: Option<u32>, - pub audio_ext: Option<String>, - pub columns: Option<u32>, - pub container: Option<String>, - pub downloader_options: Option<DownloaderOptions>, - pub dynamic_range: Option<String>, - pub ext: String, - pub filepath: Option<PathBuf>, - pub filesize: Option<u64>, - pub filesize_approx: Option<u64>, - pub format: Option<String>, - pub format_id: String, - pub format_index: Option<String>, - pub format_note: Option<String>, - pub fps: Option<f64>, - pub fragment_base_url: Option<Todo>, - pub fragments: Option<Vec<Fragment>>, - pub has_drm: Option<bool>, - pub height: Option<u32>, - pub http_headers: Option<HttpHeader>, - pub is_dash_periods: Option<bool>, - pub is_live: Option<bool>, - pub language: Option<String>, - pub language_preference: Option<i32>, - pub manifest_stream_number: Option<u32>, - pub manifest_url: Option<Url>, - pub preference: Option<i32>, - pub protocol: Option<String>, - pub quality: Option<f64>, - pub resolution: Option<String>, - pub rows: Option<u32>, - pub source_preference: Option<i32>, - pub tbr: Option<f64>, - pub url: Url, - pub vbr: Option<f64>, - pub vcodec: String, - pub video_ext: Option<String>, - pub width: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -#[allow(missing_copy_implementations)] -pub struct DownloaderOptions { - http_chunk_size: u64, -} - -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct HttpHeader { - #[serde(alias = "User-Agent")] - pub user_agent: Option<String>, - - #[serde(alias = "Accept")] - pub accept: Option<String>, - - #[serde(alias = "X-Forwarded-For")] - pub x_forwarded_for: Option<String>, - - #[serde(alias = "Accept-Language")] - pub accept_language: Option<String>, - - #[serde(alias = "Sec-Fetch-Mode")] - pub sec_fetch_mode: Option<String>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Fragment { - pub duration: Option<f64>, - pub fragment_count: Option<usize>, - pub path: Option<PathBuf>, - pub url: Option<Url>, -} - -impl InfoJson { - pub fn to_py_dict(self, py: Python<'_>) -> PyResult<Bound<'_, PyDict>> { - let output: Bound<'_, PyDict> = json_loads_str(py, self)?; - Ok(output) - } -} diff --git a/crates/yt_dlp/src/wrapper/yt_dlp_options.rs b/crates/yt_dlp/src/wrapper/yt_dlp_options.rs deleted file mode 100644 index 25595b5..0000000 --- a/crates/yt_dlp/src/wrapper/yt_dlp_options.rs +++ /dev/null @@ -1,62 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use pyo3::{Bound, PyResult, Python, types::PyDict}; -use serde::Serialize; - -use crate::json_loads; - -#[derive(Serialize, Clone)] -pub struct YtDlpOptions { - pub playliststart: u32, - pub playlistend: u32, - pub noplaylist: bool, - pub extract_flat: ExtractFlat, - // pub extractor_args: ExtractorArgs, - // pub format: String, - // pub fragment_retries: u32, - // #[serde(rename(serialize = "getcomments"))] - // pub get_comments: bool, - // #[serde(rename(serialize = "ignoreerrors"))] - // pub ignore_errors: bool, - // pub retries: u32, - // #[serde(rename(serialize = "writeinfojson"))] - // pub write_info_json: bool, - // pub postprocessors: Vec<serde_json::Map<String, serde_json::Value>>, -} - -#[derive(Serialize, Copy, Clone)] -pub enum ExtractFlat { - #[serde(rename(serialize = "in_playlist"))] - InPlaylist, - - #[serde(rename(serialize = "discard_in_playlist"))] - DiscardInPlaylist, -} - -#[derive(Serialize, Clone)] -pub struct ExtractorArgs { - pub youtube: YoutubeExtractorArgs, -} - -#[derive(Serialize, Clone)] -pub struct YoutubeExtractorArgs { - comment_sort: Vec<String>, - max_comments: Vec<String>, -} - -impl YtDlpOptions { - pub fn to_py_dict(self, py: Python) -> PyResult<Bound<PyDict>> { - let string = serde_json::to_string(&self).expect("This should always work"); - - let output: Bound<PyDict> = json_loads(py, string)?; - Ok(output) - } -} diff --git a/crates/yt_dlp/update.sh b/crates/yt_dlp/update.sh index c1a0215..ab03b62 100755 --- a/crates/yt_dlp/update.sh +++ b/crates/yt_dlp/update.sh @@ -10,6 +10,4 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -cd "$(dirname "$0")" || exit 1 -[ "$1" = "upgrade" ] && cargo upgrade --incompatible -cargo update +./crates/pyo3-pylogger/update.sh "$@" |