diff options
Diffstat (limited to '')
169 files changed, 11006 insertions, 6657 deletions
diff --git a/crates/bytes/.gitignore b/contrib/external_commands_script.sh index 8876ea6..219eae7 100644..100755 --- a/crates/bytes/.gitignore +++ b/contrib/external_commands_script.sh @@ -1,6 +1,8 @@ +#! /usr/bin/env sh + # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> # SPDX-License-Identifier: GPL-3.0-or-later # # This file is part of Yt. @@ -8,4 +10,10 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -/target +riverctl focus-output next + +alacritty --title "floating please" --command "$@" + +riverctl focus-output next + +# vim: ft=sh diff --git a/crates/bytes/Cargo.lock b/crates/bytes/Cargo.lock deleted file mode 100644 index b30ba3d..0000000 --- a/crates/bytes/Cargo.lock +++ /dev/null @@ -1,65 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "bytes" -version = "1.0.0" -dependencies = [ - "serde", -] - -[[package]] -name = "proc-macro2" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "serde" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "syn" -version = "2.0.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" diff --git a/crates/bytes/src/serde.rs b/crates/bytes/src/serde.rs deleted file mode 100644 index 4341e32..0000000 --- a/crates/bytes/src/serde.rs +++ /dev/null @@ -1,19 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use serde::{Serialize, Serializer}; - -use crate::Bytes; - -impl Serialize for Bytes { - fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { - serializer.serialize_str(self.to_string().as_str()) - } -} diff --git a/crates/bytes/Cargo.toml b/crates/colors/Cargo.toml index 4439aa8..4edefcf 100644 --- a/crates/bytes/Cargo.toml +++ b/crates/colors/Cargo.toml @@ -1,7 +1,8 @@ # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 uutils developers +# SPDX-License-Identifier: MIT # # This file is part of Yt. # @@ -9,25 +10,17 @@ # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. [package] -name = "bytes" -description = "Simple byte formatting utilities" -keywords = [] -categories = [] -version.workspace = true -edition.workspace = true +name = "colors" authors.workspace = true license.workspace = true +description = "A owo-colors inspired color crate." +version.workspace = true +edition.workspace = true repository.workspace = true rust-version.workspace = true publish = false [dependencies] -serde.workspace = true - -[dev-dependencies] [lints] workspace = true - -[package.metadata.docs.rs] -all-features = true diff --git a/crates/colors/src/custom.rs b/crates/colors/src/custom.rs new file mode 100644 index 0000000..fd6b7b3 --- /dev/null +++ b/crates/colors/src/custom.rs @@ -0,0 +1,75 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// Taken from <https://github.com/owo-colors/owo-colors/blob/61f8bba2f5f80e9f4fa600fbfdf2c21656f1d523/src/colors/custom.rs> +// at 2025-07-16T18:05:55 CEST. + +const U8_TO_STR: [[u8; 3]; 256] = generate_lookup(); + +const fn generate_lookup() -> [[u8; 3]; 256] { + let mut table = [[0, 0, 0]; 256]; + + let mut i = 0; + while i < 256 { + table[i] = [ + b'0' + (i / 100) as u8, + b'0' + (i / 10 % 10) as u8, + b'0' + (i % 10) as u8, + ]; + i += 1; + } + + table +} + +#[derive(Clone, Copy)] +pub(crate) enum Plane { + Fg, + Bg, +} + +pub(crate) const fn rgb_to_ansi(r: u8, g: u8, b: u8, plane: Plane) -> [u8; 18] { + let mut buf = *b"\x1b[p8;2;rrr;ggg;bbb"; + + let r = U8_TO_STR[r as usize]; + let g = U8_TO_STR[g as usize]; + let b = U8_TO_STR[b as usize]; + + // p 2 + buf[2] = match plane { + Plane::Fg => b'3', + Plane::Bg => b'4', + }; + + // r 7 + buf[7] = r[0]; + buf[8] = r[1]; + buf[9] = r[2]; + + // g 11 + buf[11] = g[0]; + buf[12] = g[1]; + buf[13] = g[2]; + + // b 15 + buf[15] = b[0]; + buf[16] = b[1]; + buf[17] = b[2]; + + buf +} + +/// This exists since [`unwrap()`] isn't const-safe (it invokes formatting infrastructure) +pub(crate) const fn bytes_to_str(bytes: &'static [u8]) -> &'static str { + match core::str::from_utf8(bytes) { + Ok(o) => o, + Err(_e) => panic!("Const parsing &[u8] to a string failed!"), + } +} diff --git a/crates/colors/src/lib.rs b/crates/colors/src/lib.rs new file mode 100644 index 0000000..663e19a --- /dev/null +++ b/crates/colors/src/lib.rs @@ -0,0 +1,97 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{Display, Write}; + +use crate::{ + list::{elements, methods}, + support::{CSE, CSI, elements_inner}, +}; + +pub(crate) mod custom; +mod list; +mod support; + +#[derive(Debug)] +pub struct Canvas<I: Display>(I); + +impl<I: Display> Colorize for Canvas<I> { + fn render_into(self, base: &mut String, use_colors: bool) { + write!(base, "{}", self.0).expect("Is written into a string"); + + if use_colors { + // Reset the color and style, if we used colours. + base.write_str(CSI).expect("In-memory write"); + base.write_str("0").expect("In-memory write"); + base.write_str(CSE).expect("In-memory write"); + } + } +} + +pub trait IntoCanvas: Display + Sized { + fn into_canvas(self) -> Canvas<Self> { + Canvas(self) + } + + methods! { IntoCanvas } +} + +impl<I: Display> IntoCanvas for I {} + +pub trait Colorize: Sized { + /// Turn this colorized struct into a string, by writing into the base. + fn render_into(self, base: &mut String, use_colors: bool); + + /// Turn this colorized struct into a string for consumption. + fn render(self, use_colors: bool) -> String { + let mut base = String::new(); + self.render_into(&mut base, use_colors); + base + } + + methods! { Colorize } +} + +elements! {} + +#[cfg(test)] +mod tests { + use crate::{Colorize, IntoCanvas}; + + #[test] + fn test_colorize_basic() { + let base = "Base".green().render(true); + #[rustfmt::skip] + let expected = concat!( + "\x1b[32m", + "Base", + "\x1b[0m", + ); + + assert_eq!(base.as_str(), expected); + } + + #[test] + fn test_colorize_combo() { + let base = "Base".green().on_red().bold().strike_through().render(true); + + #[rustfmt::skip] + let expected = concat!( + "\x1b[9m", // strike_through + "\x1b[1m", // bold + "\x1b[41m", // on_red + "\x1b[32m", // green + "Base", + "\x1b[0m", + ); + + assert_eq!(base.as_str(), expected); + } +} diff --git a/crates/colors/src/list.rs b/crates/colors/src/list.rs new file mode 100644 index 0000000..35fcb83 --- /dev/null +++ b/crates/colors/src/list.rs @@ -0,0 +1,233 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::support::prepend_input; + +prepend_input! { + crate::support::methods_inner as methods (($tt:tt) -> {$tt}), + crate::support::elements_inner as elements, + + <shared_input> + { + // Colors + Black black 30, + OnBlack on_black 40, + + Red red 31, + OnRed on_red 41, + + Green green 32, + OnGreen on_green 42, + + Yellow yellow 33, + OnYellow on_yellow 43, + + Blue blue 34, + OnBlue on_blue 44, + + Magenta magenta 35, + OnMagenta on_magenta 45, + + Cyan cyan 36, + OnCyan on_cyan 46, + + White white 37, + OnWhite on_white 47, + + Default default 39, + OnDefault on_default 49, + + // Bright bright colors + BrightBlack bright_black 90, + OnBrightBlack on_bright_black 100, + + BrightRed bright_red 91, + OnBrightRed on_bright_red 101, + + BrightGreen bright_green 92, + OnBrightGreen on_bright_green 102, + + BrightYellow bright_yellow 93, + OnBrightYellow on_bright_yellow 103, + + BrightBlue bright_blue 94, + OnBrightBlue on_bright_blue 104, + + BrightMagenta bright_magenta 95, + OnBrightMagenta on_bright_magenta 105, + + BrightCyan bright_cyan 96, + OnBrightCyan on_bright_cyan 106, + + BrightWhite bright_white 97, + OnBrightWhite on_bright_white 107, + + // CSS colors + // TODO(@bpeetz): Also support background colors with these values. <2025-07-16> + AliceBlue alice_blue (240, 248, 255), + AntiqueWhite antique_white (250, 235, 215), + Aqua aqua (0, 255, 255), + Aquamarine aquamarine (127, 255, 212), + Azure azure (240, 255, 255), + Beige beige (245, 245, 220), + Bisque bisque (255, 228, 196), + // Black black (0, 0, 0), + BlanchedAlmond blanched_almond (255, 235, 205), + // Blue blue (0, 0, 255), + BlueViolet blue_violet (138, 43, 226), + Brown brown (165, 42, 42), + BurlyWood burly_wood (222, 184, 135), + CadetBlue cadet_blue (95, 158, 160), + Chartreuse chartreuse (127, 255, 0), + Chocolate chocolate (210, 105, 30), + Coral coral (255, 127, 80), + CornflowerBlue cornflower_blue (100, 149, 237), + Cornsilk cornsilk (255, 248, 220), + Crimson crimson (220, 20, 60), + DarkBlue dark_blue (0, 0, 139), + DarkCyan dark_cyan (0, 139, 139), + DarkGoldenRod dark_golden_rod (184, 134, 11), + DarkGray dark_gray (169, 169, 169), + DarkGrey dark_grey (169, 169, 169), + DarkGreen dark_green (0, 100, 0), + DarkKhaki dark_khaki (189, 183, 107), + DarkMagenta dark_magenta (139, 0, 139), + DarkOliveGreen dark_olive_green (85, 107, 47), + DarkOrange dark_orange (255, 140, 0), + DarkOrchid dark_orchid (153, 50, 204), + DarkRed dark_red (139, 0, 0), + DarkSalmon dark_salmon (233, 150, 122), + DarkSeaGreen dark_sea_green (143, 188, 143), + DarkSlateBlue dark_slate_blue (72, 61, 139), + DarkSlateGray dark_slate_gray (47, 79, 79), + DarkSlateGrey dark_slate_grey (47, 79, 79), + DarkTurquoise dark_turquoise (0, 206, 209), + DarkViolet dark_violet (148, 0, 211), + DeepPink deep_pink (255, 20, 147), + DeepSkyBlue deep_sky_blue (0, 191, 255), + DimGray dim_gray (105, 105, 105), + DimGrey dim_grey (105, 105, 105), + DodgerBlue dodger_blue (30, 144, 255), + FireBrick fire_brick (178, 34, 34), + FloralWhite floral_white (255, 250, 240), + ForestGreen forest_green (34, 139, 34), + Fuchsia fuchsia (255, 0, 255), + Gainsboro gainsboro (220, 220, 220), + GhostWhite ghost_white (248, 248, 255), + Gold gold (255, 215, 0), + GoldenRod golden_rod (218, 165, 32), + Gray gray (128, 128, 128), + Grey grey (128, 128, 128), + // Green green (0, 128, 0), + GreenYellow green_yellow (173, 255, 47), + HoneyDew honey_dew (240, 255, 240), + HotPink hot_pink (255, 105, 180), + IndianRed indian_red (205, 92, 92), + Indigo indigo (75, 0, 130), + Ivory ivory (255, 255, 240), + Khaki khaki (240, 230, 140), + Lavender lavender (230, 230, 250), + LavenderBlush lavender_blush (255, 240, 245), + LawnGreen lawn_green (124, 252, 0), + LemonChiffon lemon_chiffon (255, 250, 205), + LightBlue light_blue (173, 216, 230), + LightCoral light_coral (240, 128, 128), + LightCyan light_cyan (224, 255, 255), + LightGoldenRodYellow light_golden_rod_yellow (250, 250, 210), + LightGray light_gray (211, 211, 211), + LightGrey light_grey (211, 211, 211), + LightGreen light_green (144, 238, 144), + LightPink light_pink (255, 182, 193), + LightSalmon light_salmon (255, 160, 122), + LightSeaGreen light_sea_green (32, 178, 170), + LightSkyBlue light_sky_blue (135, 206, 250), + LightSlateGray light_slate_gray (119, 136, 153), + LightSlateGrey light_slate_grey (119, 136, 153), + LightSteelBlue light_steel_blue (176, 196, 222), + LightYellow light_yellow (255, 255, 224), + Lime lime (0, 255, 0), + LimeGreen lime_green (50, 205, 50), + Linen linen (250, 240, 230), + // Magenta magenta (255, 0, 255), + Maroon maroon (128, 0, 0), + MediumAquaMarine medium_aqua_marine (102, 205, 170), + MediumBlue medium_blue (0, 0, 205), + MediumOrchid medium_orchid (186, 85, 211), + MediumPurple medium_purple (147, 112, 219), + MediumSeaGreen medium_sea_green (60, 179, 113), + MediumSlateBlue medium_slate_blue (123, 104, 238), + MediumSpringGreen medium_spring_green (0, 250, 154), + MediumTurquoise medium_turquoise (72, 209, 204), + MediumVioletRed medium_violet_red (199, 21, 133), + MidnightBlue midnight_blue (25, 25, 112), + MintCream mint_cream (245, 255, 250), + MistyRose misty_rose (255, 228, 225), + Moccasin moccasin (255, 228, 181), + NavajoWhite navajo_white (255, 222, 173), + Navy navy (0, 0, 128), + OldLace old_lace (253, 245, 230), + Olive olive (128, 128, 0), + OliveDrab olive_drab (107, 142, 35), + Orange orange (255, 165, 0), + OrangeRed orange_red (255, 69, 0), + Orchid orchid (218, 112, 214), + PaleGoldenRod pale_golden_rod (238, 232, 170), + PaleGreen pale_green (152, 251, 152), + PaleTurquoise pale_turquoise (175, 238, 238), + PaleVioletRed pale_violet_red (219, 112, 147), + PapayaWhip papaya_whip (255, 239, 213), + PeachPuff peach_puff (255, 218, 185), + Peru peru (205, 133, 63), + Pink pink (255, 192, 203), + Plum plum (221, 160, 221), + PowderBlue powder_blue (176, 224, 230), + Purple purple (128, 0, 128), + RebeccaPurple rebecca_purple (102, 51, 153), + // Red red (255, 0, 0), + RosyBrown rosy_brown (188, 143, 143), + RoyalBlue royal_blue (65, 105, 225), + SaddleBrown saddle_brown (139, 69, 19), + Salmon salmon (250, 128, 114), + SandyBrown sandy_brown (244, 164, 96), + SeaGreen sea_green (46, 139, 87), + SeaShell sea_shell (255, 245, 238), + Sienna sienna (160, 82, 45), + Silver silver (192, 192, 192), + SkyBlue sky_blue (135, 206, 235), + SlateBlue slate_blue (106, 90, 205), + SlateGray slate_gray (112, 128, 144), + SlateGrey slate_grey (112, 128, 144), + Snow snow (255, 250, 250), + SpringGreen spring_green (0, 255, 127), + SteelBlue steel_blue (70, 130, 180), + Tan tan (210, 180, 140), + Teal teal (0, 128, 128), + Thistle thistle (216, 191, 216), + Tomato tomato (255, 99, 71), + Turquoise turquoise (64, 224, 208), + Violet violet (238, 130, 238), + Wheat wheat (245, 222, 179), + // White white (255, 255, 255), + WhiteSmoke white_smoke (245, 245, 245), + // Yellow yellow (255, 255, 0), + YellowGreen yellow_green (154, 205, 50), + + // Styles + Bold bold 1, + Dim dim 2, + Italic italic 3, + Underline underline 4, + Blink blink 5, + BlinkFast blink_fast 6, + Reversed reversed 7, + Hidden hidden 8, + StrikeThrough strike_through 9, + } +} diff --git a/crates/colors/src/support.rs b/crates/colors/src/support.rs new file mode 100644 index 0000000..3c3f87d --- /dev/null +++ b/crates/colors/src/support.rs @@ -0,0 +1,126 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(super) const CSI: &str = "\x1b["; +pub(super) const CSE: &str = "m"; + +macro_rules! elements_inner { + ( + $( + $name:ident $_:ident $number:tt + ),* + $(,)? + ) => { + $( + #[derive(Debug)] + pub struct $name<I: Colorize>(I); + + impl<I: Colorize> Colorize for $name<I> { + fn render_into(self, base: &mut String, use_colors: bool) { + elements_inner! {@parse_number $number} + + if use_colors { + base.write_str(CSI).expect("In-memory write"); + base.write_str(NUMBERS).expect("In-memory write"); + base.write_str(CSE).expect("In-memory write"); + } + self.0.render_into(base, use_colors); + // The canvas is resetting the colours again. + } + } + )* + }; + + (@parse_number $single:literal) => { + const NUMBERS: &str = stringify!($single); + }; + (@parse_number ($red:literal, $green:literal, $blue:literal)) => { + const NUMBERS_U8: [u8; 18] = $crate::custom::rgb_to_ansi($red, $green, $blue, $crate::custom::Plane::Fg); + + const NUMBERS: &str = $crate::custom::bytes_to_str(&NUMBERS_U8); + } +} +pub(super) use elements_inner; + +macro_rules! methods_inner { + ( + Colorize + + $( + $struct_name:ident $fn_name:ident $_:tt + ),* + $(,)? + ) => { + $( + fn $fn_name(self) -> $struct_name<Self> { + $struct_name(self) + } + )* + }; + ( + IntoCanvas + + $( + $struct_name:ident $fn_name:ident $_:tt + ),* + $(,)? + ) => { + $( + fn $fn_name(self) -> $struct_name<Canvas<Self>> { + $struct_name(Canvas(self)) + } + )* + }; +} +pub(super) use methods_inner; + +macro_rules! prepend_input { + ( + $( + $existing_macro_name:path as $new_macro_name:ident $(($macro_rule:tt -> $macro_apply:tt))? + ),* + $(,)? + + <shared_input> + $shared_input:tt + ) => { + $( + prepend_input! { + @generate_macro + $existing_macro_name as $new_macro_name $(($macro_rule -> $macro_apply))? + <shared_input> + $shared_input + } + )* + }; + + ( + @generate_macro + $existing_macro_name:path as $new_macro_name:ident $((($($macro_rule:tt)*) -> {$($macro_apply:tt)*}))? + + <shared_input> + { + $( + $shared_input:tt + )* + } + ) => { + macro_rules! $new_macro_name { + ($($($macro_rule)*)?) => { + $existing_macro_name! { + $($($macro_apply)*)? + $($shared_input)* + } + } + } + pub(super) use $new_macro_name; + } +} +pub(crate) use prepend_input; diff --git a/crates/fmt/Cargo.toml b/crates/fmt/Cargo.toml index f3cf4ad..dd314ab 100644 --- a/crates/fmt/Cargo.toml +++ b/crates/fmt/Cargo.toml @@ -24,7 +24,7 @@ publish = false path = "src/fmt.rs" [dependencies] -unicode-width = "0.2.1" +unicode-width = "0.2.2" [lints] workspace = true diff --git a/crates/libmpv2/CHANGELOG.md b/crates/libmpv2/CHANGELOG.md index dc6f861..a3d14d7 100644 --- a/crates/libmpv2/CHANGELOG.md +++ b/crates/libmpv2/CHANGELOG.md @@ -16,7 +16,7 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. ## Version 3.0.0 -- \[breaking\] Support libmpv version 2.0 (mpv version 0.35.0). Mpv versions \<= +- [breaking] Support libmpv version 2.0 (mpv version 0.35.0). Mpv versions \<= 0.34.0 will no longer be supported. - Add OpenGL rendering @@ -29,10 +29,10 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. ## Version 2.0.0 - Add method `Mpv::with_initializer` to set options before initialization -- \[breaking\] Borrow `&mut self` in `wait_event` to disallow using two events +- [breaking] Borrow `&mut self` in `wait_event` to disallow using two events where the first points to data freed in the second `wait_event` call -- \[breaking\] `PropertyData<'_>` is no longer `Clone` or `PartialEq`, - `Event<'_>` is no longer `Clone` to avoid cloning/comparing `MpvNode` +- [breaking] `PropertyData<'_>` is no longer `Clone` or `PartialEq`, `Event<'_>` + is no longer `Clone` to avoid cloning/comparing `MpvNode` ## Version 1.1.0 diff --git a/crates/libmpv2/Cargo.toml b/crates/libmpv2/Cargo.toml index fb2f5bf..67fbfec 100644 --- a/crates/libmpv2/Cargo.toml +++ b/crates/libmpv2/Cargo.toml @@ -28,7 +28,7 @@ log.workspace = true [dev-dependencies] crossbeam = "0.8" -sdl2 = "0.37.0" +sdl2 = "0.38.0" [features] default = ["protocols", "render"] diff --git a/crates/libmpv2/libmpv2-sys/Cargo.toml b/crates/libmpv2/libmpv2-sys/Cargo.toml index 96141d3..0be2c7a 100644 --- a/crates/libmpv2/libmpv2-sys/Cargo.toml +++ b/crates/libmpv2/libmpv2-sys/Cargo.toml @@ -23,4 +23,4 @@ rust-version.workspace = true publish = false [build-dependencies] -bindgen = { version = "0.72.0" } +bindgen = { version = "0.72.1" } diff --git a/crates/libmpv2/libmpv2-sys/build.rs b/crates/libmpv2/libmpv2-sys/build.rs index bf9a02e..45c2450 100644 --- a/crates/libmpv2/libmpv2-sys/build.rs +++ b/crates/libmpv2/libmpv2-sys/build.rs @@ -30,7 +30,9 @@ fn main() { ), "--verbose", ]) - .generate_comments(true) + // NOTE(@bpeetz): The comments are interpreted as doc-tests, + // which obviously fail, as the code is c. <2025-06-16> + .generate_comments(false) .generate() .expect("Unable to generate bindings"); diff --git a/crates/libmpv2/src/mpv.rs b/crates/libmpv2/src/mpv.rs index 29dac8d..d8164c0 100644 --- a/crates/libmpv2/src/mpv.rs +++ b/crates/libmpv2/src/mpv.rs @@ -552,21 +552,21 @@ impl Mpv { /// /// # Examples /// - /// ```dont_run - /// # use libmpv2::{Mpv}; - /// # use libmpv2::mpv_node::MpvNode; - /// # use libmpv2::mpv::errors::Result; - /// # use std::collections::HashMap; - /// # - /// # fn main() -> Result<()> { - /// # let mpv = Mpv::new()?; + /// ```text + ///# use libmpv2::{Mpv}; + ///# use libmpv2::mpv_node::MpvNode; + ///# use libmpv2::mpv::errors::Result; + ///# use std::collections::HashMap; + ///# + ///# fn main() -> Result<()> { + ///# let mpv = Mpv::new()?; /// mpv.command("loadfile", &["test-data/jellyfish.mp4", "append-play"]).unwrap(); - /// # let node = mpv.get_property::<MpvNode>("playlist").unwrap(); - /// # let mut list = node.array().unwrap().collect::<Vec<_>>(); - /// # let map = list.pop().unwrap().map().unwrap().collect::<HashMap<_, _>>(); - /// # assert_eq!(map, HashMap::from([(String::from("id"), MpvNode::Int64(1)), (String::from("current"), MpvNode::Flag(true)), (String::from("filename"), MpvNode::String(String::from("test-data/jellyfish.mp4")))])); - /// # Ok(()) - /// # } + ///# let node = mpv.get_property::<MpvNode>("playlist").unwrap(); + ///# let mut list = node.array().unwrap().collect::<Vec<_>>(); + ///# let map = list.pop().unwrap().map().unwrap().collect::<HashMap<_, _>>(); + ///# assert_eq!(map, HashMap::from([(String::from("id"), MpvNode::Int64(1)), (String::from("current"), MpvNode::Flag(true)), (String::from("filename"), MpvNode::String(String::from("test-data/jellyfish.mp4")))])); + ///# Ok(()) + ///# } /// ``` pub fn command(&self, name: &str, args: &[&str]) -> Result<()> { fn escape(input: &str) -> String { diff --git a/crates/libmpv2/src/mpv/events.rs b/crates/libmpv2/src/mpv/events.rs index f10ff6e..9f6324a 100644 --- a/crates/libmpv2/src/mpv/events.rs +++ b/crates/libmpv2/src/mpv/events.rs @@ -238,7 +238,7 @@ impl EventContext { /// Returns `Some(Err(...))` if there was invalid utf-8, or if either an /// `MPV_EVENT_GET_PROPERTY_REPLY`, `MPV_EVENT_SET_PROPERTY_REPLY`, `MPV_EVENT_COMMAND_REPLY`, /// or `MPV_EVENT_PROPERTY_CHANGE` event failed, or if `MPV_EVENT_END_FILE` reported an error. - pub fn wait_event(&mut self, timeout: f64) -> Option<Result<Event>> { + pub fn wait_event(&mut self, timeout: f64) -> Option<Result<Event<'_>>> { let event = unsafe { *libmpv2_sys::mpv_wait_event(self.ctx.as_ptr(), timeout) }; // debug!("Got an event from mpv: {:#?}", event); diff --git a/crates/libmpv2/src/mpv/protocol.rs b/crates/libmpv2/src/mpv/protocol.rs index ee33411..070fb66 100644 --- a/crates/libmpv2/src/mpv/protocol.rs +++ b/crates/libmpv2/src/mpv/protocol.rs @@ -24,7 +24,7 @@ impl Mpv { /// /// # Panics /// Panics if a context already exists - pub fn create_protocol_context<T, U>(&self) -> ProtocolContext<T, U> + pub fn create_protocol_context<T, U>(&self) -> ProtocolContext<'_, T, U> where T: RefUnwindSafe, U: RefUnwindSafe, diff --git a/crates/libmpv2/update.sh b/crates/libmpv2/update.sh index ecd5aa8..e1669a9 100755 --- a/crates/libmpv2/update.sh +++ b/crates/libmpv2/update.sh @@ -10,8 +10,4 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -cd "$(dirname "$0")" || exit 1 -[ "$1" = "upgrade" ] && cargo upgrade --incompatible -cargo update - -./libmpv2-sys/update.sh "$@" +"$(dirname "$0")/libmpv2-sys/update.sh" "$@" diff --git a/crates/yt/Cargo.toml b/crates/yt/Cargo.toml index 6803e68..6184eb7 100644 --- a/crates/yt/Cargo.toml +++ b/crates/yt/Cargo.toml @@ -24,43 +24,44 @@ rust-version.workspace = true publish = false [dependencies] -anyhow = "1.0.98" -blake3 = "1.8.2" -chrono = { version = "0.4.41", features = ["now"] } +anyhow = "1.0.100" +blake3 = { version = "1.8.2", features = ["serde"] } +chrono = { version = "0.4.42", features = ["now"] } chrono-humanize = "0.2.3" -clap = { version = "4.5.40", features = ["derive"] } +clap = { version = "4.5.53", features = ["derive"] } +clap_complete = { version = "4.5.61", features = ["unstable-dynamic"] } +colors.workspace = true futures = "0.3.31" -nucleo-matcher = "0.3.1" -owo-colors = "4.2.1" -regex = "1.11.1" -sqlx = { version = "0.8.6", features = ["runtime-tokio", "sqlite"] } -stderrlog = "0.6.0" -tempfile = "3.20.0" -toml = "0.8.23" -trinitry = { version = "0.2.2" } -xdg = "3.0.0" -bytes.workspace = true libmpv2.workspace = true log.workspace = true +notify = { version = "8.2.0", default-features = false } +regex = "1.12.2" serde.workspace = true serde_json.workspace = true +shlex = "1.3.0" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "sqlite"] } +stderrlog = "0.6.0" +tempfile = "3.23.0" +termsize.workspace = true +tokio-util = { version = "0.7.17", features = ["rt"] } tokio.workspace = true +toml = "0.9.8" url.workspace = true -yt_dlp.workspace = true -termsize.workspace = true uu_fmt.workspace = true -notify = { version = "8.0.0", default-features = false } -tokio-util = { version = "0.7.15", features = ["rt"] } +xdg = "3.0.0" +yt_dlp.workspace = true +reqwest = "0.12.24" [[bin]] name = "yt" doc = false path = "src/main.rs" -[dev-dependencies] - [lints] workspace = true +[dev-dependencies] +pretty_assertions = "1.4.1" + [package.metadata.docs.rs] all-features = true diff --git a/crates/yt/src/ansi_escape_codes.rs b/crates/yt/src/ansi_escape_codes.rs index ae1805d..28a8370 100644 --- a/crates/yt/src/ansi_escape_codes.rs +++ b/crates/yt/src/ansi_escape_codes.rs @@ -1,9 +1,19 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + // see: https://en.wikipedia.org/wiki/ANSI_escape_code#Control_Sequence_Introducer_commands const CSI: &str = "\x1b["; -pub fn erase_in_display_from_cursor() { +pub(crate) fn erase_from_cursor_to_bottom() { print!("{CSI}0J"); } -pub fn cursor_up(number: usize) { +pub(crate) fn cursor_up(number: usize) { // HACK(@bpeetz): The default is `1` and running this command with a // number of `0` results in it using the default (i.e., `1`) <2025-03-25> if number != 0 { @@ -11,16 +21,9 @@ pub fn cursor_up(number: usize) { } } -pub fn clear_whole_line() { +pub(crate) fn clear_whole_line() { eprint!("{CSI}2K"); } -pub fn move_to_col(x: usize) { +pub(crate) fn move_to_col(x: usize) { eprint!("{CSI}{x}G"); } - -pub fn hide_cursor() { - eprint!("{CSI}?25l"); -} -pub fn show_cursor() { - eprint!("{CSI}?25h"); -} diff --git a/crates/yt/src/app.rs b/crates/yt/src/app.rs index 15a9388..3ea12a4 100644 --- a/crates/yt/src/app.rs +++ b/crates/yt/src/app.rs @@ -16,13 +16,13 @@ use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; use crate::{config::Config, storage::migrate::migrate_db}; #[derive(Debug)] -pub struct App { - pub database: SqlitePool, - pub config: Config, +pub(crate) struct App { + pub(crate) database: SqlitePool, + pub(crate) config: Config, } impl App { - pub async fn new(config: Config, should_migrate_db: bool) -> Result<Self> { + pub(crate) async fn new(config: Config, should_migrate_db: bool) -> Result<Self> { let options = SqliteConnectOptions::new() .filename(&config.paths.database_path) .optimize_on_close(true, None) diff --git a/crates/yt/src/cache/mod.rs b/crates/yt/src/cache/mod.rs deleted file mode 100644 index 83d5ee0..0000000 --- a/crates/yt/src/cache/mod.rs +++ /dev/null @@ -1,105 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use anyhow::{Context, Result}; -use log::{debug, info}; -use tokio::fs; - -use crate::{ - app::App, - storage::video_database::{ - Video, VideoStatus, VideoStatusMarker, downloader::set_video_cache_path, get, - }, -}; - -async fn invalidate_video(app: &App, video: &Video, hard: bool) -> Result<()> { - info!("Invalidating cache of video: '{}'", video.title); - - if hard { - if let VideoStatus::Cached { - cache_path: path, .. - } = &video.status - { - info!("Removing cached video at: '{}'", path.display()); - if let Err(err) = fs::remove_file(path).await.map_err(|err| err.kind()) { - match err { - std::io::ErrorKind::NotFound => { - // The path is already gone - debug!( - "Not actually removing path: '{}'. It is already gone.", - path.display() - ); - } - err => Err(std::io::Error::from(err)).with_context(|| { - format!( - "Failed to delete video ('{}') cache path: '{}'.", - video.title, - path.display() - ) - })?, - } - } - } - } - - set_video_cache_path(app, &video.extractor_hash, None).await?; - - Ok(()) -} - -pub async fn invalidate(app: &App, hard: bool) -> Result<()> { - let all_cached_things = get::videos(app, &[VideoStatusMarker::Cached]).await?; - - info!("Got videos to invalidate: '{}'", all_cached_things.len()); - - for video in all_cached_things { - invalidate_video(app, &video, hard).await?; - } - - Ok(()) -} - -/// # Panics -/// Only if internal assertions fail. -pub async fn maintain(app: &App, all: bool) -> Result<()> { - let domain = if all { - VideoStatusMarker::ALL.as_slice() - } else { - &[VideoStatusMarker::Watch, VideoStatusMarker::Cached] - }; - - let cached_videos = get::videos(app, domain).await?; - - let mut found_focused = 0; - for vid in cached_videos { - if let VideoStatus::Cached { - cache_path: path, - is_focused, - } = &vid.status - { - info!("Checking if path ('{}') exists", path.display()); - if !path.exists() { - invalidate_video(app, &vid, false).await?; - } - - if *is_focused { - found_focused += 1; - } - } - } - - assert!( - found_focused <= 1, - "Only one video can be focused at a time" - ); - - Ok(()) -} diff --git a/crates/yt/src/cli.rs b/crates/yt/src/cli.rs index 634e422..9a24403 100644 --- a/crates/yt/src/cli.rs +++ b/crates/yt/src/cli.rs @@ -9,390 +9,59 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use std::{path::PathBuf, str::FromStr}; +use std::path::PathBuf; -use anyhow::Context; -use bytes::Bytes; -use chrono::NaiveDate; -use clap::{ArgAction, Args, Parser, Subcommand}; -use url::Url; +use clap::{ArgAction, Parser}; -use crate::{ - select::selection_file::duration::MaybeDuration, - storage::video_database::extractor_hash::LazyExtractorHash, -}; +use crate::commands::Command; #[derive(Parser, Debug)] #[clap(author, about, long_about = None)] #[allow(clippy::module_name_repetitions)] /// An command line interface to select, download and watch videos -pub struct CliArgs { +pub(crate) struct CliArgs { #[command(subcommand)] /// The subcommand to execute [default: select] - pub command: Option<Command>, + pub(crate) command: Option<Command>, /// Show the version and exit #[arg(long, short = 'V', action= ArgAction::SetTrue)] - pub version: bool, + pub(crate) version: bool, /// Do not perform database migration before starting. /// Setting this could cause runtime database access errors. #[arg(long, short, action=ArgAction::SetTrue, default_value_t = false)] - pub no_migrate_db: bool, + pub(crate) no_migrate_db: bool, /// Display colors [defaults to true, if the config file has no value] #[arg(long, short = 'C')] - pub color: Option<bool>, + pub(crate) color: Option<bool>, /// Set the path to the videos.db. This overrides the default and the config file. #[arg(long, short)] - pub db_path: Option<PathBuf>, + pub(crate) db_path: Option<PathBuf>, /// Set the path to the config.toml. /// This overrides the default. #[arg(long, short)] - pub config_path: Option<PathBuf>, + pub(crate) config_path: Option<PathBuf>, /// Increase message verbosity #[arg(long="verbose", short = 'v', action = ArgAction::Count)] - pub verbosity: u8, + pub(crate) verbosity: u8, /// Silence all output #[arg(long, short = 'q')] - pub quiet: bool, + pub(crate) quiet: bool, } -#[derive(Subcommand, Debug)] -pub enum Command { - /// Download and cache URLs - Download { - /// Forcefully re-download all cached videos (i.e. delete the cache path, then download). - #[arg(short, long)] - force: bool, +#[cfg(test)] +mod test { + use clap::CommandFactory; - /// The maximum size the download dir should have. Beware that the value must be given in - /// bytes. - #[arg(short, long, value_parser = byte_parser)] - max_cache_size: Option<u64>, - }, - - /// Select, download and watch in one command. - Sedowa {}, - /// Download and watch in one command. - Dowa {}, - - /// Work with single videos - Videos { - #[command(subcommand)] - cmd: VideosCommand, - }, - - /// Watch the already cached (and selected) videos - Watch {}, - - /// Visualize the current playlist - Playlist { - /// Linger and display changes - #[arg(short, long)] - watch: bool, - }, - - /// Show, which videos have been selected to be watched (and their cache status) - Status {}, - - /// Show, the configuration options in effect - Config {}, - - /// Display the comments of the currently playing video - Comments {}, - /// Display the description of the currently playing video - Description {}, - - /// Manipulate the video cache in the database - #[command(visible_alias = "db")] - Database { - #[command(subcommand)] - command: CacheCommand, - }, - - /// Change the state of videos in the database (the default) - Select { - #[command(subcommand)] - cmd: Option<SelectCommand>, - }, - - /// Update the video database - Update { - /// The maximal number of videos to fetch for each subscription. - #[arg(short, long)] - max_backlog: Option<usize>, - - /// How many subs were already checked. - /// - /// Only used in the progress display in combination with `--grouped`. - #[arg(short, long, hide = true)] - current_progress: Option<usize>, - - /// How many subs are to be checked. - /// - /// Only used in the progress display in combination with `--grouped`. - #[arg(short, long, hide = true)] - total_number: Option<usize>, - - /// The subscriptions to update - subscriptions: Vec<String>, - - /// Perform the updates in blocks. - /// - /// This works around the memory leaks in the default update invocation. - #[arg( - short, - long, - conflicts_with = "total_number", - conflicts_with = "current_progress" - )] - grouped: bool, - }, - - /// Manipulate subscription - #[command(visible_alias = "subs")] - Subscriptions { - #[command(subcommand)] - cmd: SubscriptionCommand, - }, -} - -fn byte_parser(input: &str) -> Result<u64, anyhow::Error> { - Ok(input - .parse::<Bytes>() - .with_context(|| format!("Failed to parse '{input}' as bytes!"))? - .as_u64()) -} - -impl Default for Command { - fn default() -> Self { - Self::Select { - cmd: Some(SelectCommand::default()), - } + use super::CliArgs; + #[test] + fn verify_cli() { + CliArgs::command().debug_assert(); } } - -#[derive(Subcommand, Clone, Debug)] -pub enum VideosCommand { - /// List the videos in the database - #[command(visible_alias = "ls")] - List { - /// An optional search query to limit the results - #[arg(action = ArgAction::Append)] - search_query: Option<String>, - - /// The number of videos to show - #[arg(short, long)] - limit: Option<usize>, - }, - - /// Get detailed information about a video - Info { - /// The short hash of the video - hash: LazyExtractorHash, - }, -} - -#[derive(Subcommand, Clone, Debug)] -pub enum SubscriptionCommand { - /// Subscribe to an URL - Add { - #[arg(short, long)] - /// The human readable name of the subscription - name: Option<String>, - - /// The URL to listen to - url: Url, - }, - - /// Unsubscribe from an URL - Remove { - /// The human readable name of the subscription - name: String, - }, - - /// Import a bunch of URLs as subscriptions. - Import { - /// The file containing the URLs. Will use Stdin otherwise. - file: Option<PathBuf>, - - /// Remove any previous subscriptions - #[arg(short, long)] - force: bool, - }, - /// Write all subscriptions in an format understood by `import` - Export {}, - - /// List all subscriptions - List {}, -} - -#[derive(Clone, Debug, Args)] -#[command(infer_subcommands = true)] -/// Mark the video given by the hash to be watched -pub struct SharedSelectionCommandArgs { - /// The ordering priority (higher means more at the top) - #[arg(short, long)] - pub priority: Option<i64>, - - /// The subtitles to download (e.g. 'en,de,sv') - #[arg(short = 'l', long)] - pub subtitle_langs: Option<String>, - - /// The speed to set mpv to - #[arg(short, long)] - pub speed: Option<f64>, - - /// The short extractor hash - pub hash: LazyExtractorHash, - - pub title: Option<String>, - - pub date: Option<OptionalNaiveDate>, - - pub publisher: Option<OptionalPublisher>, - - pub duration: Option<MaybeDuration>, - - pub url: Option<Url>, -} -#[derive(Clone, Debug, Copy)] -pub struct OptionalNaiveDate { - pub date: Option<NaiveDate>, -} -impl FromStr for OptionalNaiveDate { - type Err = anyhow::Error; - fn from_str(v: &str) -> Result<Self, Self::Err> { - if v == "[No release date]" { - Ok(Self { date: None }) - } else { - Ok(Self { - date: Some(NaiveDate::from_str(v)?), - }) - } - } -} -#[derive(Clone, Debug)] -pub struct OptionalPublisher { - pub publisher: Option<String>, -} -impl FromStr for OptionalPublisher { - type Err = anyhow::Error; - fn from_str(v: &str) -> Result<Self, Self::Err> { - if v == "[No author]" { - Ok(Self { publisher: None }) - } else { - Ok(Self { - publisher: Some(v.to_owned()), - }) - } - } -} - -#[derive(Subcommand, Clone, Debug)] -// NOTE: Keep this in sync with the [`constants::HELP_STR`] constant. <2024-08-20> -// NOTE: Also keep this in sync with the `tree-sitter-yts/grammar.js`. <2024-11-04> -pub enum SelectCommand { - /// Open a `git rebase` like file to select the videos to watch (the default) - File { - /// Include done (watched, dropped) videos - #[arg(long, short)] - done: bool, - - /// Generate a directory, where each file contains only one subscription. - #[arg(long, short, conflicts_with = "use_last_selection")] - split: bool, - - /// Use the last selection file (useful if you've spend time on it and want to get it again) - #[arg(long, short, conflicts_with = "done")] - use_last_selection: bool, - }, - - /// Add a video to the database - /// - /// This optionally supports to add a playlist. - /// When a playlist is added, the `start` and `stop` arguments can be used to select which - /// playlist entries to include. - #[command(visible_alias = "a")] - Add { - urls: Vec<Url>, - - /// Start adding playlist entries at this playlist index (zero based and inclusive) - #[arg(short = 's', long)] - start: Option<usize>, - - /// Stop adding playlist entries at this playlist index (zero based and inclusive) - #[arg(short = 'e', long)] - stop: Option<usize>, - }, - - /// Mark the video given by the hash to be watched - #[command(visible_alias = "w")] - Watch { - #[command(flatten)] - shared: SharedSelectionCommandArgs, - }, - - /// Mark the video given by the hash to be dropped - #[command(visible_alias = "d")] - Drop { - #[command(flatten)] - shared: SharedSelectionCommandArgs, - }, - - /// Mark the video given by the hash as already watched - #[command(visible_alias = "wd")] - Watched { - #[command(flatten)] - shared: SharedSelectionCommandArgs, - }, - - /// Open the video URL in Firefox's `timesinks.youtube` profile - #[command(visible_alias = "u")] - Url { - #[command(flatten)] - shared: SharedSelectionCommandArgs, - }, - - /// Reset the videos status to 'Pick' - #[command(visible_alias = "p")] - Pick { - #[command(flatten)] - shared: SharedSelectionCommandArgs, - }, -} -impl Default for SelectCommand { - fn default() -> Self { - Self::File { - done: false, - use_last_selection: false, - split: false, - } - } -} - -#[derive(Subcommand, Clone, Copy, Debug)] -pub enum CacheCommand { - /// Invalidate all cache entries - Invalidate { - /// Also delete the cache path - #[arg(short, long)] - hard: bool, - }, - - /// Perform basic maintenance operations on the database. - /// This helps recovering from invalid db states after a crash (or force exit via <CTRL-C>). - /// - /// 1. Check every path for validity (removing all invalid cache entries) - #[command(verbatim_doc_comment)] - Maintain { - /// Check every video (otherwise only the videos to be watched are checked) - #[arg(short, long)] - all: bool, - }, -} diff --git a/crates/yt/src/commands/cache/implm.rs b/crates/yt/src/commands/cache/implm.rs new file mode 100644 index 0000000..fd0fbce --- /dev/null +++ b/crates/yt/src/commands/cache/implm.rs @@ -0,0 +1,40 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::CacheCommand, + storage::db::{ + insert::Operations, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +impl CacheCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + CacheCommand::Clear {} => { + let mut videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("Cache clear"); + + for vid in &mut videos { + vid.remove_download_path(&mut ops); + } + + ops.commit(app).await?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/cache/mod.rs b/crates/yt/src/commands/cache/mod.rs new file mode 100644 index 0000000..4ed4b40 --- /dev/null +++ b/crates/yt/src/commands/cache/mod.rs @@ -0,0 +1,19 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Subcommand; + +mod implm; + +#[derive(Debug, Subcommand)] +pub(super) enum CacheCommand { + /// Remove all downloaded video files. + Clear {}, +} diff --git a/crates/yt/src/commands/config/implm.rs b/crates/yt/src/commands/config/implm.rs new file mode 100644 index 0000000..00c28a9 --- /dev/null +++ b/crates/yt/src/commands/config/implm.rs @@ -0,0 +1,23 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{app::App, commands::config::ConfigCommand}; + +use anyhow::Result; + +impl ConfigCommand { + pub(in crate::commands) fn implm(self, app: &App) -> Result<()> { + let config_str = toml::to_string(&app.config)?; + + print!("{config_str}"); + + Ok(()) + } +} diff --git a/crates/yt/src/constants.rs b/crates/yt/src/commands/config/mod.rs index 0f5b918..503b4f7 100644 --- a/crates/yt/src/constants.rs +++ b/crates/yt/src/commands/config/mod.rs @@ -1,6 +1,5 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> // Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // @@ -9,4 +8,9 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -pub const HELP_STR: &str = include_str!("./select/selection_file/help.str"); +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct ConfigCommand {} diff --git a/crates/yt/src/commands/database/implm.rs b/crates/yt/src/commands/database/implm.rs new file mode 100644 index 0000000..07d346b --- /dev/null +++ b/crates/yt/src/commands/database/implm.rs @@ -0,0 +1,45 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::DatabaseCommand, + storage::db::{ + insert::{Committable, subscription, video}, + txn_log::TxnLog, + }, +}; + +use anyhow::Result; + +impl DatabaseCommand { + pub(in crate::commands) async fn implm(&self, app: &App) -> Result<()> { + match self { + DatabaseCommand::Log { kind } => match kind { + super::OperationType::Video => { + let log = TxnLog::<video::Operation>::get(app).await?; + display_log(&log); + } + super::OperationType::Subscription => { + let log = TxnLog::<subscription::Operation>::get(app).await?; + display_log(&log); + } + }, + } + + Ok(()) + } +} + +fn display_log<O: Committable>(log: &TxnLog<O>) { + for (time, value) in log.inner() { + println!("At {time}: {value:?}"); + } +} diff --git a/crates/yt/src/commands/database/mod.rs b/crates/yt/src/commands/database/mod.rs new file mode 100644 index 0000000..06e3169 --- /dev/null +++ b/crates/yt/src/commands/database/mod.rs @@ -0,0 +1,41 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{self, Display}; + +use clap::{Subcommand, ValueEnum}; + +mod implm; + +#[derive(Subcommand, Debug)] +pub(super) enum DatabaseCommand { + /// Show the history of operations, in they groups they were committed in. + Log { + /// What kind of operation to show. + #[arg(short, long, default_value_t)] + kind: OperationType, + }, +} + +#[derive(Debug, Clone, Copy, ValueEnum, Default)] +pub(super) enum OperationType { + #[default] + Video, + Subscription, +} + +impl Display for OperationType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OperationType::Video => f.write_str("video"), + OperationType::Subscription => f.write_str("subscription"), + } + } +} diff --git a/crates/yt/src/download/download_options.rs b/crates/yt/src/commands/download/implm/download/download_options.rs index 03c20ba..15fed7e 100644 --- a/crates/yt/src/download/download_options.rs +++ b/crates/yt/src/commands/download/implm/download/download_options.rs @@ -11,13 +11,16 @@ use anyhow::Context; use serde_json::{Value, json}; -use yt_dlp::{YoutubeDL, YoutubeDLOptions}; +use yt_dlp::{YoutubeDL, options::YoutubeDLOptions}; -use crate::{app::App, storage::video_database::YtDlpOptions}; +use crate::app::App; use super::progress_hook::wrapped_progress_hook; -pub fn download_opts(app: &App, additional_opts: &YtDlpOptions) -> anyhow::Result<YoutubeDL> { +pub(crate) fn download_opts( + app: &App, + subtitle_langs: Option<&String>, +) -> anyhow::Result<YoutubeDL> { YoutubeDLOptions::new() .with_progress_hook(wrapped_progress_hook) .set("extract_flat", "in_playlist") @@ -106,8 +109,8 @@ pub fn download_opts(app: &App, additional_opts: &YtDlpOptions) -> anyhow::Resul .set( "subtitleslangs", Value::Array( - additional_opts - .subtitle_langs + subtitle_langs + .map_or("", String::as_str) .split(',') .map(|val| Value::String(val.to_owned())) .collect::<Vec<_>>(), diff --git a/crates/yt/src/commands/download/implm/download/mod.rs b/crates/yt/src/commands/download/implm/download/mod.rs new file mode 100644 index 0000000..ab9de80 --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/mod.rs @@ -0,0 +1,290 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; + +use crate::{ + app::App, + commands::download::implm::download::download_options::download_opts, + shared::bytes::Bytes, + storage::{ + db::{extractor_hash::ExtractorHash, insert::Operations, video::Video}, + notify::{wait_for_cache_reduction, wait_for_db_write}, + }, + yt_dlp::get_current_cache_allocation, +}; + +use anyhow::{Context, Result, bail}; +use log::{debug, error, info, warn}; +use tokio::{select, task::JoinHandle, time}; +use yt_dlp::YoutubeDL; + +#[allow(clippy::module_name_repetitions)] +pub(crate) mod download_options; +pub(crate) mod progress_hook; + +#[derive(Debug)] +#[allow(clippy::module_name_repetitions)] +pub(crate) struct CurrentDownload { + task_handle: JoinHandle<Result<(PathBuf, Video)>>, + yt_dlp: Arc<YoutubeDL>, + extractor_hash: ExtractorHash, +} + +impl CurrentDownload { + fn new_from_video(app: &App, video: Video) -> Result<Self> { + let extractor_hash = video.extractor_hash; + + debug!("Download started: {}", &video.title); + let yt_dlp = Arc::new(download_opts(app, video.subtitle_langs.as_ref())?); + + let local_yt_dlp = Arc::clone(&yt_dlp); + + let task_handle = tokio::task::spawn_blocking(move || { + let mut result = local_yt_dlp + .download(&[video.url.clone()]) + .with_context(|| format!("Failed to download video: '{}'", video.title))?; + + assert_eq!(result.len(), 1); + Ok((result.remove(0), video)) + }); + + Ok(Self { + task_handle, + yt_dlp, + extractor_hash, + }) + } + + fn abort(self) -> Result<()> { + debug!("Cancelling download."); + self.yt_dlp.close()?; + + Ok(()) + } + + fn is_finished(&self) -> bool { + self.task_handle.is_finished() + } + + async fn finalize(self, app: &App) -> Result<()> { + let (result, mut video) = self.task_handle.await??; + + let mut ops = Operations::new("Downloader: Set download path"); + video.set_download_path(&result, &mut ops); + ops.commit(app) + .await + .with_context(|| format!("Failed to commit download of video: '{}'", video.title))?; + + info!( + "Video '{}' was downlaoded to path: {}", + video.title, + result.display() + ); + + Ok(()) + } +} + +enum CacheSizeCheck { + /// The video can be downloaded + Fits, + + /// The video and the current cache size together would exceed the size + TooLarge, + + /// The video would not even fit into the empty cache + ExceedsMaxCacheSize, +} + +#[derive(Debug)] +pub(crate) struct Downloader { + current_download: Option<CurrentDownload>, + video_size_cache: HashMap<ExtractorHash, u64>, + printed_warning: bool, + cached_cache_allocation: Option<Bytes>, +} + +impl Default for Downloader { + fn default() -> Self { + Self::new() + } +} + +impl Downloader { + #[must_use] + pub(crate) fn new() -> Self { + Self { + current_download: None, + video_size_cache: HashMap::new(), + printed_warning: false, + cached_cache_allocation: None, + } + } + + /// Check if enough cache is available. + /// + /// Will wait for the next cache deletion if not. + async fn is_enough_cache_available( + &mut self, + app: &App, + max_cache_size: u64, + next_video: &Video, + ) -> Result<CacheSizeCheck> { + if let Some(cdownload) = &self.current_download { + if cdownload.extractor_hash == next_video.extractor_hash { + // If the video is already being downloaded it will always fit. Otherwise the + // download would not have been started. + return Ok(CacheSizeCheck::Fits); + } + } + let cache_allocation = get_current_cache_allocation(app).await?; + let video_size = self.get_approx_video_size(next_video)?; + + if video_size >= max_cache_size { + error!( + "The video '{}' ({}) exceeds the maximum cache size ({})! \ + Please set a bigger maximum (`--max-cache-size`) or skip it.", + next_video.title, + Bytes::new(video_size), + Bytes::new(max_cache_size) + ); + + return Ok(CacheSizeCheck::ExceedsMaxCacheSize); + } + + if cache_allocation.as_u64() + video_size >= max_cache_size { + if !self.printed_warning { + warn!( + "Can't download video: '{}' ({}) as it's too large for the cache ({} of {} allocated). \ + Waiting for cache size reduction..", + next_video.title, + Bytes::new(video_size), + &cache_allocation, + Bytes::new(max_cache_size) + ); + self.printed_warning = true; + + // Update this value immediately. + // This avoids printing the "Current cache size has changed .." warning below. + self.cached_cache_allocation = Some(cache_allocation); + } + + if let Some(cca) = self.cached_cache_allocation { + if cca != cache_allocation { + // Only print the warning if the display string has actually changed. + // Otherwise, we might confuse the user + if cca.to_string() != cache_allocation.to_string() { + warn!("Current cache size has changed, it's now: '{cache_allocation}'"); + } + debug!( + "Cache size has changed: {} -> {}", + cca.as_u64(), + cache_allocation.as_u64() + ); + self.cached_cache_allocation = Some(cache_allocation); + } + } else { + unreachable!( + "The `printed_warning` should be false in this case, \ + and thus should have already set the `cached_cache_allocation`." + ); + } + + // Wait and hope, that a large video is deleted from the cache. + wait_for_cache_reduction(app).await?; + Ok(CacheSizeCheck::TooLarge) + } else { + self.printed_warning = false; + Ok(CacheSizeCheck::Fits) + } + } + + /// The entry point to the Downloader. + /// This Downloader will periodically check if the database has changed, and then also + /// change which videos it downloads. + /// This will run, until the database doesn't contain any watchable videos + pub(crate) async fn consume(&mut self, app: Arc<App>, max_cache_size: u64) -> Result<()> { + while let Some(next_video) = Video::next_to_download(&app).await? { + match self + .is_enough_cache_available(&app, max_cache_size, &next_video) + .await? + { + CacheSizeCheck::Fits => (), + CacheSizeCheck::TooLarge => continue, + CacheSizeCheck::ExceedsMaxCacheSize => bail!("Giving up."), + } + + if self.current_download.is_some() { + let current_download = self.current_download.take().expect("It is `Some`."); + + if current_download.is_finished() { + // The download is done, finalize it and leave it removed. + current_download.finalize(&app).await?; + continue; + } + + if next_video.extractor_hash == current_download.extractor_hash { + // We still want to download the same video. + // reset the taken value + self.current_download = Some(current_download); + } else { + info!( + "Noticed, that the next video is not the video being downloaded, replacing it ('{}' vs. '{}')!", + next_video.extractor_hash.as_short_hash(&app).await?, + current_download.extractor_hash.as_short_hash(&app).await? + ); + + // Replace the currently downloading video + current_download + .abort() + .context("Failed to abort last download")?; + + let new_current_download = CurrentDownload::new_from_video(&app, next_video)?; + + self.current_download = Some(new_current_download); + } + } else { + info!( + "No video is being downloaded right now, setting it to '{}'", + next_video.title + ); + let new_current_download = CurrentDownload::new_from_video(&app, next_video)?; + self.current_download = Some(new_current_download); + } + + // We have to continuously check, if the current download is done. + // As such we simply wait or recheck on the next write to the db. + select! { + () = time::sleep(Duration::from_secs(1)) => (), + Ok(()) = wait_for_db_write(&app) => (), + } + } + + info!("Finished downloading!"); + Ok(()) + } + + fn get_approx_video_size(&mut self, video: &Video) -> Result<u64> { + if let Some(value) = self.video_size_cache.get(&video.extractor_hash) { + Ok(*value) + } else { + let size = video.get_approx_size()?; + + assert_eq!( + self.video_size_cache.insert(video.extractor_hash, size), + None + ); + + Ok(size) + } + } +} diff --git a/crates/yt/src/commands/download/implm/download/progress_hook.rs b/crates/yt/src/commands/download/implm/download/progress_hook.rs new file mode 100644 index 0000000..a414d4a --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/progress_hook.rs @@ -0,0 +1,210 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::HashSet, + io::{Write, stderr}, + process, + sync::{Mutex, atomic::Ordering}, +}; + +use colors::{Colorize, IntoCanvas}; +use log::{Level, log_enabled}; +use yt_dlp::{json_cast, json_get, wrap_progress_hook}; + +use crate::{ + ansi_escape_codes::{clear_whole_line, move_to_col}, + config::SHOULD_DISPLAY_COLOR, + select::duration::MaybeDuration, + shared::bytes::Bytes, +}; + +macro_rules! json_get_default { + ($value:expr, $name:literal, $convert:ident, $default:expr) => { + $value.get($name).map_or($default, |v| { + if v == &serde_json::Value::Null { + $default + } else { + json_cast!(@log_key $name, v, $convert) + } + }) + }; +} + +static TITLES: Mutex<Option<HashSet<String>>> = Mutex::new(None); + +fn format_bytes(bytes: u64) -> String { + let bytes = Bytes::new(bytes); + bytes.to_string() +} + +fn format_speed(speed: f64) -> String { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let bytes = Bytes::new(speed.floor() as u64); + format!("{bytes}/s") +} + +/// # Panics +/// If expectations fail. +#[allow(clippy::needless_pass_by_value)] +pub(crate) fn progress_hook( + input: serde_json::Map<String, serde_json::Value>, +) -> Result<(), std::io::Error> { + // Only add the handler, if the log-level is higher than Debug (this avoids covering debug + // messages). + if log_enabled!(Level::Debug) { + return Ok(()); + } + + let info_dict = json_get!(input, "info_dict", as_object); + + let get_title = || -> String { + match json_get!(info_dict, "ext", as_str) { + "vtt" => { + format!( + "Subtitles ({})", + json_get_default!(info_dict, "name", as_str, "<No Subtitle Language>") + ) + } + "webm" | "mp4" | "mp3" | "m4a" => { + json_get_default!(info_dict, "title", as_str, "<No title>").to_owned() + } + other => panic!("The extension '{other}' is not yet implemented"), + } + }; + + match json_get!(input, "status", as_str) { + "downloading" => { + let elapsed = json_get_default!(input, "elapsed", as_f64, 0.0); + let eta = json_get_default!(input, "eta", as_f64, 0.0); + let speed = json_get_default!(input, "speed", as_f64, 0.0); + + let downloaded_bytes = json_get!(input, "downloaded_bytes", as_u64); + let (total_bytes, bytes_is_estimate): (u64, &'static str) = { + let total_bytes = json_get_default!(input, "total_bytes", as_u64, 0); + + if total_bytes == 0 { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let maybe_estimate = + json_get_default!(input, "total_bytes_estimate", as_f64, 0.0) as u64; + + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + if maybe_estimate == 0 { + // The download speed should be in bytes + // per second and the eta in seconds. + // Thus multiplying them gets us the raw bytes + // (which were estimated by `yt_dlp`, from their `info.json`) + let bytes_still_needed = (speed * eta).ceil() as u64; + + (downloaded_bytes + bytes_still_needed, "~") + } else { + (maybe_estimate, "~") + } + } else { + (total_bytes, "") + } + }; + + let percent: f64 = { + if total_bytes == 0 { + 100.0 + } else { + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + { + (downloaded_bytes as f64 / total_bytes as f64) * 100.0 + } + } + }; + + clear_whole_line(); + move_to_col(1); + + let should_use_color = SHOULD_DISPLAY_COLOR.load(Ordering::Relaxed); + + let title = get_title(); + + eprint!( + "{} [{}/{} at {}] -> [{} of {}{} {}] ", + (&title).bold().blue().render(should_use_color), + MaybeDuration::from_secs_f64(elapsed) + .bold() + .yellow() + .render(should_use_color), + MaybeDuration::from_secs_f64(eta) + .bold() + .yellow() + .render(should_use_color), + format_speed(speed).bold().green().render(should_use_color), + format_bytes(downloaded_bytes) + .bold() + .red() + .render(should_use_color), + bytes_is_estimate.bold().red().render(should_use_color), + format_bytes(total_bytes) + .bold() + .red() + .render(should_use_color), + format!("{percent:.02}%") + .bold() + .cyan() + .render(should_use_color), + ); + stderr().flush()?; + + { + let mut titles = TITLES.lock().expect("The lock should work"); + + match titles.as_mut() { + Some(titles) => { + titles.insert(title); + } + None => *titles = Some(HashSet::from_iter([title])), + } + } + } + "finished" => { + let should_use_color = SHOULD_DISPLAY_COLOR.load(Ordering::Relaxed); + let title = get_title(); + + let has_already_been_printed = { + let titles = TITLES.lock().expect("The lock should work"); + + match titles.as_ref() { + Some(titles) => titles.contains(&title), + None => false, + } + }; + + if has_already_been_printed { + eprintln!("-> Finished downloading."); + } else { + eprintln!( + "Download of {} already finished.", + title.bold().blue().render(should_use_color) + ); + } + } + "error" => { + // TODO: This should probably return an Err. But I'm not so sure where the error would + // bubble up to (i.e., who would catch it) <2025-01-21> + eprintln!("-> Error while downloading: {}", get_title()); + process::exit(1); + } + other => unreachable!("'{other}' should not be a valid state!"), + } + + Ok(()) +} + +wrap_progress_hook!(progress_hook, wrapped_progress_hook); diff --git a/crates/yt/src/commands/download/implm/mod.rs b/crates/yt/src/commands/download/implm/mod.rs new file mode 100644 index 0000000..c74a909 --- /dev/null +++ b/crates/yt/src/commands/download/implm/mod.rs @@ -0,0 +1,55 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::Arc; + +use crate::{ + app::App, + commands::download::DownloadCommand, + shared::bytes::Bytes, + storage::db::{ + insert::{Operations, maintenance::clear_stale_downloaded_paths}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; +use log::info; + +mod download; + +impl DownloadCommand { + pub(in crate::commands) async fn implm(self, app: Arc<App>) -> Result<()> { + let DownloadCommand { + force, + max_cache_size, + } = self; + + let max_cache_size = max_cache_size.unwrap_or(app.config.download.max_cache_size.as_u64()); + info!("Max cache size: '{}'", Bytes::new(max_cache_size)); + + clear_stale_downloaded_paths(&app).await?; + if force { + let mut all = Video::in_states(&app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("Download: Clear old download paths due to `--force`"); + for a in &mut all { + a.remove_download_path(&mut ops); + } + ops.commit(&app).await?; + } + + download::Downloader::new() + .consume(app, max_cache_size) + .await?; + + Ok(()) + } +} diff --git a/crates/yt/src/commands/download/mod.rs b/crates/yt/src/commands/download/mod.rs new file mode 100644 index 0000000..15026ba --- /dev/null +++ b/crates/yt/src/commands/download/mod.rs @@ -0,0 +1,34 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Context; +use clap::Parser; + +use crate::shared::bytes::Bytes; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct DownloadCommand { + /// Forcefully re-download all cached videos (i.e. delete all already downloaded paths, then download). + #[arg(short, long)] + force: bool, + + /// The maximum size the download dir should have. + #[arg(short, long, value_parser = byte_parser)] + max_cache_size: Option<u64>, +} + +fn byte_parser(input: &str) -> Result<u64, anyhow::Error> { + Ok(input + .parse::<Bytes>() + .with_context(|| format!("Failed to parse '{input}' as bytes!"))? + .as_u64()) +} diff --git a/crates/yt/src/commands/implm.rs b/crates/yt/src/commands/implm.rs new file mode 100644 index 0000000..7c60c6a --- /dev/null +++ b/crates/yt/src/commands/implm.rs @@ -0,0 +1,38 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::Arc; + +use crate::commands::Command; + +use anyhow::Result; + +impl Command { + pub(crate) async fn implm(self, app: crate::app::App) -> Result<()> { + match self { + Command::Cache { cmd } => cmd.implm(&app).await?, + Command::Config { cmd } => cmd.implm(&app)?, + Command::Database { cmd } => cmd.implm(&app).await?, + Command::Download { cmd } => cmd.implm(Arc::new(app)).await?, + Command::Playlist { cmd } => cmd.implm(&app).await?, + Command::Select { cmd } => { + cmd.unwrap_or_default().implm(&app).await?; + } + Command::Show { cmd } => cmd.implm(&app).await?, + Command::Status { cmd } => cmd.implm(&app).await?, + Command::Subscriptions { cmd } => cmd.implm(&app).await?, + Command::Update { cmd } => cmd.implm(&app).await?, + Command::Videos { cmd } => cmd.implm(&app).await?, + Command::Watch { cmd } => cmd.implm(Arc::new(app)).await?, + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/mod.rs b/crates/yt/src/commands/mod.rs new file mode 100644 index 0000000..431acef --- /dev/null +++ b/crates/yt/src/commands/mod.rs @@ -0,0 +1,164 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ffi::OsStr, thread}; + +use clap::Subcommand; +use clap_complete::CompletionCandidate; +use tokio::runtime::Runtime; + +use crate::{ + app::App, + commands::{ + cache::CacheCommand, config::ConfigCommand, database::DatabaseCommand, + download::DownloadCommand, playlist::PlaylistCommand, select::SelectCommand, + show::ShowCommand, status::StatusCommand, subscriptions::SubscriptionCommand, + update::UpdateCommand, videos::VideosCommand, watch::WatchCommand, + }, + config::Config, + storage::db::subscription::Subscriptions, +}; + +pub(super) mod implm; + +mod cache; +mod config; +mod database; +mod download; +mod playlist; +mod select; +mod show; +mod status; +mod subscriptions; +mod update; +mod videos; +mod watch; + +#[derive(Subcommand, Debug)] +#[allow(private_interfaces)] // Only the main `implm` method should be accessible. +pub(super) enum Command { + /// Manipulate the download cache + Cache { + #[command(subcommand)] + cmd: CacheCommand, + }, + + /// Show, the configuration options in effect. + Config { + #[command(flatten)] + cmd: ConfigCommand, + }, + + /// Interact with the video database. + #[command(visible_alias = "db")] + Database { + #[command(subcommand)] + cmd: DatabaseCommand, + }, + + /// Download and cache URLs + Download { + #[command(flatten)] + cmd: DownloadCommand, + }, + + /// Visualize the current playlist + Playlist { + #[command(flatten)] + cmd: PlaylistCommand, + }, + + /// Change the state of videos in the database (the default) + Select { + #[command(subcommand)] + cmd: Option<SelectCommand>, + }, + + /// Show things about the currently playing video. + Show { + #[command(subcommand)] + cmd: ShowCommand, + }, + + /// Show, which videos have been selected to be watched (and their cache status) + Status { + #[command(flatten)] + cmd: StatusCommand, + }, + + /// Manipulate subscription + #[command(visible_alias = "subs")] + Subscriptions { + #[command(subcommand)] + cmd: SubscriptionCommand, + }, + + /// Update the video database + Update { + #[command(flatten)] + cmd: UpdateCommand, + }, + + /// Work with single videos + Videos { + #[command(subcommand)] + cmd: VideosCommand, + }, + + /// Watch the already cached (and selected) videos + Watch { + #[command(flatten)] + cmd: WatchCommand, + }, +} + +impl Default for Command { + fn default() -> Self { + Self::Select { + cmd: Some(SelectCommand::default()), + } + } +} + +fn complete_subscription(current: &OsStr) -> Vec<CompletionCandidate> { + let mut output = vec![]; + + let Some(current_prog) = current.to_str().map(ToOwned::to_owned) else { + return output; + }; + + let Ok(config) = Config::from_config_file(None, None, None) else { + return output; + }; + + let handle = thread::spawn(move || { + let Ok(rt) = Runtime::new() else { + return output; + }; + + let Ok(app) = rt.block_on(App::new(config, false)) else { + return output; + }; + + let Ok(all) = rt.block_on(Subscriptions::get(&app)) else { + return output; + }; + + for sub in all.0.into_keys() { + if sub.starts_with(¤t_prog) { + output.push(CompletionCandidate::new(sub)); + } + } + + output + }); + + handle.join().unwrap_or_default() +} diff --git a/crates/yt/src/commands/playlist/implm.rs b/crates/yt/src/commands/playlist/implm.rs new file mode 100644 index 0000000..603184b --- /dev/null +++ b/crates/yt/src/commands/playlist/implm.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{fmt::Write, path::Path}; + +use crate::{ + ansi_escape_codes, + app::App, + commands::playlist::PlaylistCommand, + storage::{ + db::{ + playlist::Playlist, + video::{Video, VideoStatus}, + }, + notify::wait_for_db_write, + }, + videos::RenderWithApp, +}; + +use anyhow::Result; +use futures::{TryStreamExt, stream::FuturesOrdered}; + +impl PlaylistCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let PlaylistCommand { watch } = self; + + let mut previous_output_length = 0; + loop { + let playlist = Playlist::create(app).await?.videos; + + let output = playlist + .into_iter() + .map(|video| async move { + let mut output = String::new(); + + let (_, is_focused) = cache_values(&video); + + if is_focused { + output.push_str("🔻 "); + } else { + output.push_str(" "); + } + + output.push_str(&video.title_fmt().to_string(app)); + + output.push_str(" ("); + output.push_str(&video.parent_subscription_name_fmt().to_string(app)); + output.push(')'); + + output.push_str(" ["); + output.push_str(&video.duration_fmt().to_string(app)); + + if is_focused { + output.push_str(" ("); + if let Some(percent) = video.watch_progress_percent_fmt() { + write!(output, "{}", percent.to_string(app))?; + } else { + write!(output, "{}", video.watch_progress_fmt().to_string(app))?; + } + + output.push(')'); + } + output.push(']'); + + output.push('\n'); + + Ok::<String, anyhow::Error>(output) + }) + .collect::<FuturesOrdered<_>>() + .try_collect::<String>() + .await?; + + // Delete the previous output + ansi_escape_codes::cursor_up(previous_output_length); + ansi_escape_codes::erase_from_cursor_to_bottom(); + + previous_output_length = output.chars().filter(|ch| *ch == '\n').count(); + + print!("{output}"); + + if !watch { + break; + } + + wait_for_db_write(app).await?; + } + + Ok(()) + } +} + +/// Extract the values of the [`VideoStatus::Cached`] value from a Video. +fn cache_values(video: &Video) -> (&Path, bool) { + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + { + (cache_path, *is_focused) + } else { + unreachable!("All of these videos should be cached"); + } +} diff --git a/crates/yt/src/commands/playlist/mod.rs b/crates/yt/src/commands/playlist/mod.rs new file mode 100644 index 0000000..8d3407d --- /dev/null +++ b/crates/yt/src/commands/playlist/mod.rs @@ -0,0 +1,20 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct PlaylistCommand { + /// Linger and display changes + #[arg(short, long)] + watch: bool, +} diff --git a/crates/yt/src/select/selection_file/help.str b/crates/yt/src/commands/select/implm/fs_generators/help.str index e3cc347..e3cc347 100644 --- a/crates/yt/src/select/selection_file/help.str +++ b/crates/yt/src/commands/select/implm/fs_generators/help.str diff --git a/crates/yt/src/select/selection_file/help.str.license b/crates/yt/src/commands/select/implm/fs_generators/help.str.license index a0e196c..a0e196c 100644 --- a/crates/yt/src/select/selection_file/help.str.license +++ b/crates/yt/src/commands/select/implm/fs_generators/help.str.license diff --git a/crates/yt/src/commands/select/implm/fs_generators/mod.rs b/crates/yt/src/commands/select/implm/fs_generators/mod.rs new file mode 100644 index 0000000..10da032 --- /dev/null +++ b/crates/yt/src/commands/select/implm/fs_generators/mod.rs @@ -0,0 +1,355 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::HashMap, + env, + fs::{self, File, OpenOptions}, + io::{BufRead, BufReader, BufWriter, Read, Write}, + iter, + os::fd::{AsFd, AsRawFd}, + path::Path, +}; + +use crate::{ + app::App, + cli::CliArgs, + commands::{ + Command, + select::{ + SelectCommand, SelectSplitSortKey, SelectSplitSortMode, + implm::standalone::{self, handle_select_cmd}, + }, + }, + storage::db::{ + extractor_hash::ExtractorHash, + insert::Operations, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use futures::{TryStreamExt, stream::FuturesOrdered}; +use log::info; +use shlex::Shlex; +use tokio::process; + +const HELP_STR: &str = include_str!("./help.str"); + +pub(crate) async fn select_split( + app: &App, + done: bool, + sort_key: SelectSplitSortKey, + sort_mode: SelectSplitSortMode, +) -> Result<()> { + let temp_dir = tempfile::Builder::new() + .prefix("yt_video_select-") + .rand_bytes(6) + .tempdir() + .context("Failed to get tempdir")?; + + let matching_videos = get_videos(app, done).await?; + + let mut no_author = vec![]; + let mut author_map = HashMap::new(); + for video in matching_videos { + if let Some(sub) = &video.parent_subscription_name { + if author_map.contains_key(sub) { + let vec: &mut Vec<_> = author_map + .get_mut(sub) + .expect("This key is set, we checked in the if above"); + + vec.push(video); + } else { + author_map.insert(sub.to_owned(), vec![video]); + } + } else { + no_author.push(video); + } + } + + let author_map = { + let mut temp_vec: Vec<_> = author_map.into_iter().collect(); + + match sort_key { + SelectSplitSortKey::Publisher => { + // PERFORMANCE: The clone here should not be neeed. <2025-06-15> + temp_vec.sort_by_key(|(name, _): &(String, Vec<Video>)| name.to_owned()); + } + SelectSplitSortKey::Videos => { + temp_vec.sort_by_key(|(_, videos): &(String, Vec<Video>)| videos.len()); + } + } + + match sort_mode { + SelectSplitSortMode::Asc => { + // Std's default mode is ascending. + } + SelectSplitSortMode::Desc => { + temp_vec.reverse(); + } + } + + temp_vec + }; + + for (index, (name, videos)) in author_map + .into_iter() + .chain(iter::once(( + "<No parent subscription>".to_owned(), + no_author, + ))) + .enumerate() + { + let mut file_path = temp_dir.path().join(format!("{index:02}_{name}")); + file_path.set_extension("yts"); + + let tmp_file = File::create(&file_path) + .with_context(|| format!("Falied to create file at: {}", file_path.display()))?; + + write_videos_to_file(app, &tmp_file, &videos) + .await + .with_context(|| format!("Falied to populate file at: {}", file_path.display()))?; + } + + open_editor_at(temp_dir.path()).await?; + + let mut paths = vec![]; + for maybe_entry in temp_dir + .path() + .read_dir() + .context("Failed to open temp dir for reading")? + { + let entry = maybe_entry.context("Failed to read entry in temp dir")?; + + if !entry.file_type()?.is_file() { + bail!("Found non-file entry: {}", entry.path().display()); + } + + paths.push(entry.path()); + } + + paths.sort(); + + let mut persistent_file = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(&app.config.paths.last_selection_path) + .context("Failed to open persistent selection file")?; + + for path in paths { + let mut read_file = File::open(path)?; + + let mut buffer = vec![]; + read_file.read_to_end(&mut buffer)?; + persistent_file.write_all(&buffer)?; + } + + persistent_file.flush()?; + let persistent_file = OpenOptions::new() + .read(true) + .open(format!( + "/proc/self/fd/{}", + persistent_file.as_fd().as_raw_fd() + )) + .context("Failed to re-open persistent file")?; + + let processed = process_file(app, &persistent_file).await?; + + info!("Processed {processed} records."); + temp_dir.close().context("Failed to close the temp dir")?; + Ok(()) +} + +pub(crate) async fn select_file(app: &App, done: bool, use_last_selection: bool) -> Result<()> { + let temp_file = tempfile::Builder::new() + .prefix("yt_video_select-") + .suffix(".yts") + .rand_bytes(6) + .tempfile() + .context("Failed to get tempfile")?; + + if use_last_selection { + fs::copy(&app.config.paths.last_selection_path, &temp_file)?; + } else { + let matching_videos = get_videos(app, done).await?; + + write_videos_to_file(app, temp_file.as_file(), &matching_videos).await?; + } + + open_editor_at(temp_file.path()).await?; + + let read_file = OpenOptions::new().read(true).open(temp_file.path())?; + fs::copy(temp_file.path(), &app.config.paths.last_selection_path) + .context("Failed to persist selection file")?; + + let processed = process_file(app, &read_file).await?; + info!("Processed {processed} records."); + + Ok(()) +} + +async fn get_videos(app: &App, include_done: bool) -> Result<Vec<Video>> { + if include_done { + Video::in_states(app, VideoStatusMarker::ALL).await + } else { + Video::in_states( + app, + &[ + VideoStatusMarker::Pick, + // + VideoStatusMarker::Watch, + VideoStatusMarker::Cached, + ], + ) + .await + } +} + +async fn write_videos_to_file(app: &App, file: &File, videos: &[Video]) -> Result<()> { + // Warm-up the cache for the display rendering of the videos. + // Otherwise the futures would all try to warm it up at the same time. + if let Some(vid) = videos.first() { + drop(vid.to_line_display(app, None).await?); + } + + let mut edit_file = BufWriter::new(file); + + videos + .iter() + .map(|vid| vid.to_select_file_display(app)) + .collect::<FuturesOrdered<_>>() + .try_collect::<Vec<String>>() + .await? + .into_iter() + .try_for_each(|line| -> Result<()> { + edit_file + .write_all(line.as_bytes()) + .context("Failed to write to `edit_file`")?; + + Ok(()) + })?; + + edit_file.write_all(HELP_STR.as_bytes())?; + edit_file.flush().context("Failed to flush edit file")?; + + Ok(()) +} + +async fn process_file(app: &App, file: &File) -> Result<i64> { + let mut line_number = 0; + + let mut ops = Operations::new("Select: process file"); + + // Fetch all the hashes once, instead of every time we need to process a line. + let all_hashes = ExtractorHash::get_all(app).await?; + + let reader = BufReader::new(file); + for line in reader.lines() { + let line = line.context("Failed to read a line")?; + + if let Some(line) = process_line(&line)? { + line_number -= 1; + + // debug!( + // "Parsed command: `{}`", + // line.iter() + // .map(|val| format!("\"{}\"", val)) + // .collect::<Vec<String>>() + // .join(" ") + // ); + + let arg_line = ["yt", "select"] + .into_iter() + .chain(line.iter().map(String::as_str)); + + let args = CliArgs::parse_from(arg_line); + + let Command::Select { cmd } = args + .command + .expect("This will be some, as we constructed it above.") + else { + unreachable!("This is checked in the `filter_line` function") + }; + + match cmd.expect( + "This value should always be some \ + here, as it would otherwise thrown an error above.", + ) { + SelectCommand::File { .. } | SelectCommand::Split { .. } => { + bail!("You cannot use `select file` or `select split` recursively.") + } + SelectCommand::Add { urls, start, stop } => { + Box::pin(standalone::add::add(app, urls, start, stop)).await?; + } + other => { + let shared = other + .clone() + .into_shared() + .expect("The ones without shared should have been filtered out."); + + let hash = shared.hash.realize(app, Some(&all_hashes)).await?; + let mut video = hash + .get_with_app(app) + .await + .expect("The hash was already realized, it should therefore exist"); + + handle_select_cmd(app, other, &mut video, Some(line_number), &mut ops).await?; + } + } + } + } + + ops.commit(app).await?; + Ok(-line_number) +} + +async fn open_editor_at(path: &Path) -> Result<()> { + let editor = env::var("EDITOR").unwrap_or("nvim".to_owned()); + + let mut nvim = process::Command::new(&editor); + nvim.arg(path); + let status = nvim + .status() + .await + .with_context(|| format!("Falied to run editor: {editor}"))?; + + if status.success() { + Ok(()) + } else { + bail!("Editor ({editor}) exited with error status: {}", status) + } +} + +fn process_line(line: &str) -> Result<Option<Vec<String>>> { + // Filter out comments and empty lines + if line.starts_with('#') || line.trim().is_empty() { + Ok(None) + } else { + let split: Vec<_> = { + let mut shl = Shlex::new(line); + let res = shl.by_ref().collect(); + + if shl.had_error { + bail!("Failed to parse line '{line}'") + } + + assert_eq!(shl.line_no, 1, "A unexpected newline appeared"); + res + }; + + assert!(!split.is_empty()); + + Ok(Some(split)) + } +} diff --git a/crates/yt/src/commands/select/implm/mod.rs b/crates/yt/src/commands/select/implm/mod.rs new file mode 100644 index 0000000..f39c77f --- /dev/null +++ b/crates/yt/src/commands/select/implm/mod.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{app::App, commands::select::SelectCommand, storage::db::insert::Operations}; + +use anyhow::Result; + +mod fs_generators; +mod standalone; + +impl SelectCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + SelectCommand::File { + done, + use_last_selection, + } => Box::pin(fs_generators::select_file(app, done, use_last_selection)).await?, + SelectCommand::Split { + done, + sort_key, + sort_mode, + } => Box::pin(fs_generators::select_split(app, done, sort_key, sort_mode)).await?, + SelectCommand::Add { urls, start, stop } => { + Box::pin(standalone::add::add(app, urls, start, stop)).await?; + } + other => { + let shared = other + .clone() + .into_shared() + .expect("The ones without shared should have been filtered out."); + let hash = shared.hash.realize(app, None).await?; + let mut video = hash + .get_with_app(app) + .await + .expect("The hash was already realized, it should therefore exist"); + + let mut ops = Operations::new("Main: handle select cmd"); + standalone::handle_select_cmd(app, other, &mut video, None, &mut ops).await?; + ops.commit(app).await?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/select/cmds/add.rs b/crates/yt/src/commands/select/implm/standalone/add.rs index 387b3a1..2a7db53 100644 --- a/crates/yt/src/select/cmds/add.rs +++ b/crates/yt/src/commands/select/implm/standalone/add.rs @@ -10,20 +10,17 @@ use crate::{ app::App, - download::download_options::download_opts, - storage::video_database::{ - self, extractor_hash::ExtractorHash, get::get_all_hashes, set::add_video, - }, - update::video_entry_to_video, + storage::db::{extractor_hash::ExtractorHash, insert::Operations, video::Video}, + yt_dlp::yt_dlp_opts_updating, }; use anyhow::{Context, Result, bail}; -use log::{error, warn}; +use log::{debug, error, warn}; use url::Url; -use yt_dlp::{InfoJson, YoutubeDL, json_cast, json_get}; +use yt_dlp::{YoutubeDL, info_json::InfoJson, json_cast, json_get, json_try_get}; #[allow(clippy::too_many_lines)] -pub(super) async fn add( +pub(crate) async fn add( app: &App, urls: Vec<Url>, start: Option<usize>, @@ -31,11 +28,17 @@ pub(super) async fn add( ) -> Result<()> { for url in urls { async fn process_and_add(app: &App, entry: InfoJson, yt_dlp: &YoutubeDL) -> Result<()> { - let url = json_get!(entry, "url", as_str).parse()?; - - let entry = yt_dlp - .extract_info(&url, false, true) - .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; + let entry = if json_try_get!(entry, "formats", as_array).is_some() { + // We assume, that this entry is already processed. + debug!("Refusing to re-process entry again"); + entry + } else { + let url = json_get!(entry, "url", as_str).parse()?; + + yt_dlp + .extract_info(&url, false, true) + .with_context(|| format!("Failed to fetch entry for url: '{url}'"))? + }; add_entry(app, entry).await?; @@ -45,50 +48,46 @@ pub(super) async fn add( async fn add_entry(app: &App, entry: InfoJson) -> Result<()> { // We have to re-fetch all hashes every time, because a user could try to add the same // URL twice (for whatever reason.) - let hashes = get_all_hashes(app) + let hashes = ExtractorHash::get_all(app) .await .context("Failed to fetch all video hashes")?; - let extractor_hash = blake3::hash(json_get!(entry, "id", as_str).as_bytes()); + + let extractor_hash = ExtractorHash::from_info_json(&entry); if hashes.contains(&extractor_hash) { error!( "Video '{}'{} is already in the database. Skipped adding it", - ExtractorHash::from_hash(extractor_hash) - .into_short_hash(app) + extractor_hash + .as_short_hash(app) .await .with_context(|| format!( "Failed to format hash of video '{}' as short hash", - entry - .get("url") - .map_or("<Unknown video Url>".to_owned(), ToString::to_string) + json_try_get!(entry, "url", as_str).unwrap_or("<Unknown video Url>") ))?, - entry - .get("title") - .map_or(String::new(), |title| format!(" ('{title}')")) + json_try_get!(entry, "title", as_str) + .map_or(String::new(), |title| format!(" (\"{title}\")")) ); return Ok(()); } - let video = video_entry_to_video(&entry, None)?; - add_video(app, video.clone()).await?; + let mut ops = Operations::new("SelectAdd: Video entry to video"); + let video = Video::from_info_json(&entry, None)?.add(&mut ops)?; + ops.commit(app).await?; - println!("{}", &video.to_line_display(app).await?); + println!("{}", &video.to_line_display(app, None).await?); Ok(()) } - let yt_dlp = download_opts( - app, - &video_database::YtDlpOptions { - subtitle_langs: String::new(), - }, + let yt_dlp = yt_dlp_opts_updating( + start.unwrap_or(0) + stop.map_or(usize::MAX, |val| val.saturating_add(1)), )?; let entry = yt_dlp .extract_info(&url, false, true) .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; - match entry.get("_type").map(|val| json_cast!(val, as_str)) { - Some("Video") => { + match json_try_get!(entry, "_type", as_str) { + Some("video") => { add_entry(app, entry).await?; if start.is_some() || stop.is_some() { warn!( @@ -96,12 +95,15 @@ pub(super) async fn add( ); } } - Some("Playlist") => { - if let Some(entries) = entry.get("entries") { - let entries = json_cast!(entries, as_array); + Some("playlist") => { + if let Some(entries) = json_try_get!(entry, "entries", as_array) { let start = start.unwrap_or(0); let stop = stop.unwrap_or(entries.len() - 1); + if entries.is_empty() { + bail!("Failed to add playlist, as it is empty (contains no entries).") + } + let respected_entries = take_vector(entries, start, stop).with_context(|| { format!( @@ -141,8 +143,7 @@ pub(super) async fn add( } } other => bail!( - "Your URL should point to a video or a playlist, but points to a '{:#?}'", - other + "Your URL should point to a video or a playlist, but points to a '{other:#?}'" ), } } @@ -164,7 +165,7 @@ fn take_vector<T>(vector: &[T], start: usize, stop: usize) -> Result<&[T]> { #[cfg(test)] mod test { - use crate::select::cmds::add::take_vector; + use super::take_vector; #[test] fn test_vector_take() { diff --git a/crates/yt/src/commands/select/implm/standalone/mod.rs b/crates/yt/src/commands/select/implm/standalone/mod.rs new file mode 100644 index 0000000..9512e32 --- /dev/null +++ b/crates/yt/src/commands/select/implm/standalone/mod.rs @@ -0,0 +1,132 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::io::{Write, stderr}; + +use crate::{ + ansi_escape_codes, + app::App, + commands::select::{SelectCommand, SharedSelectionCommandArgs}, + storage::db::{ + insert::{Operations, video::Operation}, + video::{Priority, Video, VideoStatus}, + }, +}; + +use anyhow::{Context, Result, bail}; + +pub(super) mod add; + +pub(crate) async fn handle_select_cmd( + app: &App, + cmd: SelectCommand, + video: &mut Video, + line_number: Option<i64>, + ops: &mut Operations<Operation>, +) -> Result<()> { + let status = match cmd { + SelectCommand::Pick { shared } => Some((VideoStatus::Pick, shared)), + SelectCommand::Drop { shared } => Some((VideoStatus::Drop, shared)), + SelectCommand::Watched { shared } => Some((VideoStatus::Watched, shared)), + SelectCommand::Watch { shared } => { + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + { + Some(( + VideoStatus::Cached { + cache_path: cache_path.to_owned(), + is_focused: *is_focused, + }, + shared, + )) + } else { + Some((VideoStatus::Watch, shared)) + } + } + SelectCommand::Url { shared } => { + let Some(url) = shared.url else { + bail!("You need to provide a url to `select url ..`") + }; + + let mut firefox = std::process::Command::new(app.config.commands.url_opener.first()); + firefox.args(app.config.commands.url_opener.tail()); + firefox.arg(url.as_str()); + let _handle = firefox.spawn().context("Failed to run firefox")?; + None + } + SelectCommand::File { .. } | SelectCommand::Split { .. } | SelectCommand::Add { .. } => { + unreachable!("These should have been filtered out") + } + }; + + if let Some((status, shared)) = status { + handle_status_change( + app, + video, + shared, + line_number, + status, + line_number.is_none(), + ops, + ) + .await?; + } + + Ok(()) +} + +async fn handle_status_change( + app: &App, + video: &mut Video, + shared: SharedSelectionCommandArgs, + line_number: Option<i64>, + new_status: VideoStatus, + is_single: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + let priority = compute_priority(line_number, shared.priority); + + video.set_status(new_status, ops); + if let Some(priority) = priority { + video.set_priority(priority, ops); + } + + if let Some(subtitle_langs) = shared.subtitle_langs { + video.set_subtitle_langs(subtitle_langs, ops); + } + if let Some(playback_speed) = shared.playback_speed { + video.set_playback_speed(playback_speed, ops); + } + + if !is_single { + ansi_escape_codes::clear_whole_line(); + ansi_escape_codes::move_to_col(1); + } + + eprint!("{}", &video.to_line_display(app, None).await?); + + if is_single { + eprintln!(); + } else { + stderr().flush()?; + } + + Ok(()) +} + +fn compute_priority(line_number: Option<i64>, priority: Option<i64>) -> Option<Priority> { + if let Some(pri) = priority { + Some(Priority::from(pri)) + } else { + line_number.map(Priority::from) + } +} diff --git a/crates/yt/src/commands/select/mod.rs b/crates/yt/src/commands/select/mod.rs new file mode 100644 index 0000000..db69238 --- /dev/null +++ b/crates/yt/src/commands/select/mod.rs @@ -0,0 +1,230 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fmt::{self, Display, Formatter}, + str::FromStr, +}; + +use chrono::NaiveDate; +use clap::{Args, Subcommand, ValueEnum}; +use url::Url; + +use crate::{select::duration::MaybeDuration, storage::db::extractor_hash::LazyExtractorHash}; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +// NOTE: Keep this in sync with the [`constants::HELP_STR`] constant. <2024-08-20> +// NOTE: Also keep this in sync with the `tree-sitter-yts/grammar.js`. <2024-11-04> +#[allow(private_interfaces)] // Only the main `implm` method should be accessible. +pub(super) enum SelectCommand { + /// Open a `git rebase` like file to select the videos to watch (the default) + File { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Use the last selection file (useful if you've spend time on it and want to get it again) + #[arg(long, short, conflicts_with = "done")] + use_last_selection: bool, + }, + + /// Generate a directory, where each file contains only one subscription. + Split { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Which key to use for sorting. + #[arg(default_value_t)] + sort_key: SelectSplitSortKey, + + /// Which mode to use for sorting. + #[arg(default_value_t)] + sort_mode: SelectSplitSortMode, + }, + + /// Add a video to the database + /// + /// This optionally supports to add a playlist. + /// When a playlist is added, the `start` and `stop` arguments can be used to select which + /// playlist entries to include. + #[command(visible_alias = "a")] + Add { + urls: Vec<Url>, + + /// Start adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 's', long)] + start: Option<usize>, + + /// Stop adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 'e', long)] + stop: Option<usize>, + }, + + /// Mark the video given by the hash to be watched + #[command(visible_alias = "w")] + Watch { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash to be dropped + #[command(visible_alias = "d")] + Drop { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash as already watched + #[command(visible_alias = "wd")] + Watched { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Open the video URL in your specified command + #[command(visible_alias = "u")] + Url { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Reset the videos status to 'Pick' + #[command(visible_alias = "p")] + Pick { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, +} +impl Default for SelectCommand { + fn default() -> Self { + Self::File { + done: false, + use_last_selection: false, + } + } +} + +#[derive(Clone, Debug, Args)] +#[command(infer_subcommands = true)] +/// Mark the video given by the hash to be watched +struct SharedSelectionCommandArgs { + /// The ordering priority (higher means more at the top) + #[arg(short, long)] + priority: Option<i64>, + + /// The subtitles to download (e.g. 'en,de,sv') + #[arg(short = 'l', long)] + subtitle_langs: Option<String>, + + /// The speed to set mpv to + #[arg(short = 's', long)] + playback_speed: Option<f64>, + + /// The short extractor hash + hash: LazyExtractorHash, + + title: Option<String>, + + date: Option<OptionalNaiveDate>, + + publisher: Option<OptionalPublisher>, + + duration: Option<MaybeDuration>, + + url: Option<Url>, +} + +impl SelectCommand { + fn into_shared(self) -> Option<SharedSelectionCommandArgs> { + match self { + SelectCommand::File { .. } + | SelectCommand::Split { .. } + | SelectCommand::Add { .. } => None, + SelectCommand::Watch { shared } + | SelectCommand::Drop { shared } + | SelectCommand::Watched { shared } + | SelectCommand::Url { shared } + | SelectCommand::Pick { shared } => Some(shared), + } + } +} + +#[derive(Clone, Debug, Copy)] +struct OptionalNaiveDate { + date: Option<NaiveDate>, +} +impl FromStr for OptionalNaiveDate { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No release date]" { + Ok(Self { date: None }) + } else { + Ok(Self { + date: Some(NaiveDate::from_str(v)?), + }) + } + } +} +#[derive(Clone, Debug)] +struct OptionalPublisher { + publisher: Option<String>, +} +impl FromStr for OptionalPublisher { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No author]" { + Ok(Self { publisher: None }) + } else { + Ok(Self { + publisher: Some(v.to_owned()), + }) + } + } +} + +#[derive(Default, ValueEnum, Clone, Copy, Debug)] +enum SelectSplitSortKey { + /// Sort by the name of the publisher. + #[default] + Publisher, + + /// Sort by the number of unselected videos per publisher. + Videos, +} +impl Display for SelectSplitSortKey { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + SelectSplitSortKey::Publisher => f.write_str("publisher"), + SelectSplitSortKey::Videos => f.write_str("videos"), + } + } +} + +#[derive(Default, ValueEnum, Clone, Copy, Debug)] +enum SelectSplitSortMode { + /// Sort in ascending order (small -> big) + #[default] + Asc, + + /// Sort in descending order (big -> small) + Desc, +} + +impl Display for SelectSplitSortMode { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + SelectSplitSortMode::Asc => f.write_str("asc"), + SelectSplitSortMode::Desc => f.write_str("desc"), + } + } +} diff --git a/crates/yt/src/commands/show/implm/mod.rs b/crates/yt/src/commands/show/implm/mod.rs new file mode 100644 index 0000000..a2e40fd --- /dev/null +++ b/crates/yt/src/commands/show/implm/mod.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fs::{self, OpenOptions}, + io, + process::Command, +}; + +use crate::{ + app::App, + commands::ShowCommand, + output::{display_fmt_and_less, display_less}, + storage::db::video::Video, +}; + +use anyhow::{Context, Result, anyhow, bail}; +use tempfile::Builder; +use tokio_util::bytes::Buf; + +impl ShowCommand { + pub(in crate::commands) async fn implm(&self, app: &App) -> Result<()> { + match self { + ShowCommand::Description {} => { + let description = Video::get_current_description(app).await?; + + display_fmt_and_less(&description)?; + } + ShowCommand::Comments {} => { + let comments = Video::get_current_comments(app).await?; + + display_less(comments.render(app.config.global.display_colors))?; + } + ShowCommand::Thumbnail {} => { + let video = Video::currently_focused(app).await?.ok_or(anyhow!( + "You need to have a current video to display its info" + ))?; + + if let Some(url) = video.thumbnail_url { + let response = reqwest::get(url.clone()) + .await + .with_context(|| format!("Failed to download thumbnail from url: {url}"))?; + let response = response + .error_for_status() + .context("Failed to download thumbnail")?; + + let (tmp_path, mut tmp) = { + let file = Builder::new().prefix("yt-thumbnail-download").tempfile()?; + let (_, path) = file.keep()?; + let new_file = OpenOptions::new() + .write(true) + .read(false) + .create(false) + .truncate(true) + .open(&path)?; + + (path, new_file) + }; + + let mut content = response.bytes().await?.reader(); + io::copy(&mut content, &mut tmp)?; + + let status = Command::new(app.config.commands.image_show.first()) + .args(app.config.commands.image_show.tail()) + .arg(tmp_path.as_os_str()) + .status() + .context("Failed to spawn image show command")?; + + if !status.success() { + bail!( + "{:?} failed with status: {}", + &app.config.commands.image_show.join(" "), + status + ); + } + + fs::remove_file(&tmp_path).with_context(|| { + format!( + "Failed to cleanup downloaded thumbnail image at: {}", + tmp_path.display() + ) + })?; + } else { + eprintln!("Current video does not have a thumbnail."); + } + } + ShowCommand::Info {} => { + let video = Video::currently_focused(app).await?.ok_or(anyhow!( + "You need to have a current video to display its info" + ))?; + + display_less( + video + .to_info_display(app, None) + .await + .context("Failed to format video")?, + )?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/show/mod.rs b/crates/yt/src/commands/show/mod.rs new file mode 100644 index 0000000..60f2e51 --- /dev/null +++ b/crates/yt/src/commands/show/mod.rs @@ -0,0 +1,30 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Subcommand; + +mod implm; + +#[derive(Subcommand, Debug)] +pub(super) enum ShowCommand { + /// Display the description of the currently playing video + Description {}, + + /// Display the comments of the currently playing video. + Comments {}, + + /// Display the thumbnail of the currently playing video. + Thumbnail {}, + + /// Display general info of the currently playing video. + /// + /// This is the same as running `yt videos info <hash of current video>` + Info {}, +} diff --git a/crates/yt/src/commands/status/implm.rs b/crates/yt/src/commands/status/implm.rs new file mode 100644 index 0000000..5832fde --- /dev/null +++ b/crates/yt/src/commands/status/implm.rs @@ -0,0 +1,166 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::time::Duration; + +use crate::{ + app::App, + commands::status::StatusCommand, + select::duration::MaybeDuration, + shared::bytes::Bytes, + storage::db::{ + subscription::Subscriptions, + video::{Video, VideoStatusMarker}, + }, + yt_dlp::get_current_cache_allocation, +}; + +use anyhow::{Context, Result}; + +macro_rules! get { + ($videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .count() + }; + + (@collect $videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .collect() + }; +} + +impl StatusCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let StatusCommand { format } = self; + + let all_videos = Video::in_states(app, VideoStatusMarker::ALL).await?; + + // lengths + let picked_videos_len = get!(all_videos, Pick); + + let watch_videos_len = get!(all_videos, Watch); + let cached_videos_len = get!(all_videos, Cached); + let watched_videos_len = get!(all_videos, Watched); + let watched_videos: Vec<_> = get!(@collect all_videos, Watched); + + let drop_videos_len = get!(all_videos, Drop); + let dropped_videos_len = get!(all_videos, Dropped); + + let subscriptions = Subscriptions::get(app).await?; + let active_subscriptions_len = subscriptions + .0 + .iter() + .filter(|(_, sub)| sub.is_active) + .count(); + let subscriptions_len = subscriptions.0.len(); + + let watchtime_status = { + let total_watch_time_raw = watched_videos + .iter() + .fold(Duration::default(), |acc, vid| acc + vid.watch_progress); + + // Most things are watched at a speed of s (which is defined in the config file). + // Thus + // y = x * s -> y / s = x + let total_watch_time = Duration::from_secs_f64( + (total_watch_time_raw.as_secs_f64()) / app.config.select.playback_speed, + ); + + let speed = app.config.select.playback_speed; + + // Do not print the adjusted time, if the user has keep the speed level at 1. + #[allow(clippy::float_cmp)] + if speed == 1.0 { + format!( + "Total Watchtime: {}\n", + MaybeDuration::from_std(total_watch_time_raw) + ) + } else { + format!( + "Total Watchtime: {} (at {speed} speed: {})\n", + MaybeDuration::from_std(total_watch_time_raw), + MaybeDuration::from_std(total_watch_time), + ) + } + }; + + let watch_rate: f64 = { + fn to_f64(input: usize) -> f64 { + f64::from(u32::try_from(input).expect("This should never exceed u32::MAX")) + } + + let count = + to_f64(watched_videos_len) / (to_f64(drop_videos_len) + to_f64(dropped_videos_len)); + count * 100.0 + }; + + let cache_usage: Bytes = get_current_cache_allocation(app) + .await + .context("Failed to get current cache allocation")?; + + if let Some(fmt) = format { + let output = fmt + .replace( + "{picked_videos_len}", + picked_videos_len.to_string().as_str(), + ) + .replace("{watch_videos_len}", watch_videos_len.to_string().as_str()) + .replace( + "{cached_videos_len}", + cached_videos_len.to_string().as_str(), + ) + .replace( + "{watched_videos_len}", + watched_videos_len.to_string().as_str(), + ) + .replace("{watch_rate}", watch_rate.to_string().as_str()) + .replace("{drop_videos_len}", drop_videos_len.to_string().as_str()) + .replace( + "{dropped_videos_len}", + dropped_videos_len.to_string().as_str(), + ) + .replace("{watchtime_status}", watchtime_status.to_string().as_str()) + .replace( + "{subscriptions_len}", + subscriptions_len.to_string().as_str(), + ) + .replace( + "{active_subscriptions_len}", + active_subscriptions_len.to_string().as_str(), + ) + .replace("{cache_usage}", cache_usage.to_string().as_str()); + + print!("{output}"); + } else { + println!( + "\ +Picked Videos: {picked_videos_len} + +Watch Videos: {watch_videos_len} +Cached Videos: {cached_videos_len} +Watched Videos: {watched_videos_len} (watch rate: {watch_rate:.2} %) + +Drop Videos: {drop_videos_len} +Dropped Videos: {dropped_videos_len} + +{watchtime_status} + + Subscriptions: {subscriptions_len} ({active_subscriptions_len} active) + Cache usage: {cache_usage}" + ); + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/status/mod.rs b/crates/yt/src/commands/status/mod.rs new file mode 100644 index 0000000..4a8dee7 --- /dev/null +++ b/crates/yt/src/commands/status/mod.rs @@ -0,0 +1,20 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct StatusCommand { + /// Which format to use + #[arg(short, long)] + format: Option<String>, +} diff --git a/crates/yt/src/commands/subscriptions/implm.rs b/crates/yt/src/commands/subscriptions/implm.rs new file mode 100644 index 0000000..1e2e545 --- /dev/null +++ b/crates/yt/src/commands/subscriptions/implm.rs @@ -0,0 +1,287 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::str::FromStr; + +use crate::{ + app::App, + commands::subscriptions::{SubscriptionCommand, SubscriptionStatus}, + storage::db::{ + insert::{Operations, subscription::Operation}, + subscription::{Subscription, Subscriptions, check_url}, + }, +}; + +use anyhow::{Context, Result, bail}; +use log::{error, warn}; +use tokio::{ + fs::File, + io::{AsyncBufRead, AsyncBufReadExt, BufReader, stdin}, +}; +use url::Url; +use yt_dlp::{json_cast, json_get, json_try_get, options::YoutubeDLOptions}; + +impl SubscriptionCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + SubscriptionCommand::Add { + name, + url, + no_check, + } => { + let mut ops = Operations::new("main: subscribe"); + subscribe(app, name, url, no_check, &mut ops) + .await + .context("Failed to add a subscription")?; + ops.commit(app).await?; + } + SubscriptionCommand::Remove { name } => { + let mut present_subscriptions = Subscriptions::get(app).await?; + + let mut ops = Operations::new("Subscribe: unsubscribe"); + if let Some(subscription) = present_subscriptions.0.remove(&name) { + subscription.remove(&mut ops); + } else { + bail!("Couldn't find subscription: '{}'", &name); + } + ops.commit(app) + .await + .with_context(|| format!("Failed to unsubscribe from {name:?}"))?; + } + SubscriptionCommand::SetStatus { name, new_status } => { + let mut present_subscriptions = Subscriptions::get(app).await?; + + let mut ops = Operations::new("Subscribe: Set Status"); + if let Some(subscription) = present_subscriptions.0.remove(&name) { + subscription.set_is_active( + match new_status { + SubscriptionStatus::Active => true, + SubscriptionStatus::Inactive => false, + }, + &mut ops, + ); + } else { + bail!("Couldn't find subscription: '{}'", &name); + } + ops.commit(app) + .await + .with_context(|| format!("Failed to change status of {name:?}"))?; + } + SubscriptionCommand::List { active } => { + let all_subs = Subscriptions::get(app).await?; + + let all_subs = if active { + all_subs.remove_inactive() + } else { + all_subs + }; + + for (key, val) in all_subs.0 { + println!( + "{}: '{}' ({})", + key, + val.url, + if val.is_active { "active" } else { "inactive" } + ); + } + } + SubscriptionCommand::Export {} => { + let all_subs = Subscriptions::get(app).await?; + for val in all_subs.0.values() { + println!("{}", val.url); + } + } + SubscriptionCommand::Import { + file, + force, + no_check, + } => { + if let Some(file) = file { + let f = File::open(file).await?; + + import(app, BufReader::new(f), force, no_check).await?; + } else { + import(app, BufReader::new(stdin()), force, no_check).await?; + } + } + } + + Ok(()) + } +} + +async fn import<W: AsyncBufRead + AsyncBufReadExt + Unpin>( + app: &App, + reader: W, + force: bool, + no_check: bool, +) -> Result<()> { + let mut ops = Operations::new("SubscribeImport: init"); + + let all = Subscriptions::get(app).await?; + if force { + all.remove(&mut ops); + } + ops.commit(app).await?; + let mut ops = Operations::new("SubscribeImport: after all subs remove"); + + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + let url = + Url::from_str(&line).with_context(|| format!("Failed to parse '{line}' as url"))?; + + match subscribe(app, None, url, no_check, &mut ops) + .await + .with_context(|| format!("Failed to subscribe to: '{line}'")) + { + Ok(()) => (), + Err(err) => eprintln!( + "Error while subscribing to '{}': '{}'", + line, + err.source().expect("Should have a source") + ), + } + } + ops.commit(app).await?; + + Ok(()) +} + +async fn subscribe( + app: &App, + name: Option<String>, + url: Url, + no_check: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + if !(url.as_str().ends_with("videos") + || url.as_str().ends_with("streams") + || url.as_str().ends_with("shorts") + || url.as_str().ends_with("videos/") + || url.as_str().ends_with("streams/") + || url.as_str().ends_with("shorts/")) + && url.as_str().contains("youtube.com") + { + warn!( + "Your youtube url does not seem like it actually tracks a channels playlist \ + (videos, streams, shorts). Adding subscriptions for each of them..." + ); + + let url = Url::parse(&(url.as_str().to_owned() + "/")) + .expect("This was an url, it should stay one"); + + let (videos, streams, shorts) = if let Some(name) = name { + ( + Some(name.clone() + " {Videos}"), + Some(name.clone() + " {Streams}"), + Some(name.clone() + " {Shorts}"), + ) + } else { + (None, None, None) + }; + + let _ = actual_subscribe( + app, + videos, + url.join("videos/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!("Failed to subscribe to the '{}' variant: {err}", "{Videos}"); + }); + + let _ = actual_subscribe( + app, + streams, + url.join("streams/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!( + "Failed to subscribe to the '{}' variant: {err}", + "{Streams}" + ); + }); + + let _ = actual_subscribe( + app, + shorts, + url.join("shorts/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!("Failed to subscribe to the '{}' variant: {err}", "{Shorts}"); + }); + } else { + actual_subscribe(app, name, url, no_check, ops).await?; + } + + Ok(()) +} + +async fn actual_subscribe( + app: &App, + name: Option<String>, + url: Url, + no_check: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + if !no_check && !check_url(url.clone()).await? { + bail!("The url ('{}') does not represent a playlist!", &url) + } + + let name = if let Some(name) = name { + name + } else { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + if json_try_get!(info, "_type", as_str) == Some("playlist") { + json_get!(info, "title", as_str).to_owned() + } else { + bail!("The url ('{}') does not represent a playlist!", &url) + } + }; + + let present_subscriptions = Subscriptions::get(app).await?; + + if let Some(subs) = present_subscriptions.0.get(&name) { + bail!( + "The subscription '{}' could not be added, \ + as another one with the same name ('{}') already exists. \ + It points to the Url: '{}'", + name, + name, + subs.url + ); + } + + let sub = Subscription { + name, + url, + is_active: true, + }; + + sub.add(ops); + + Ok(()) +} diff --git a/crates/yt/src/commands/subscriptions/mod.rs b/crates/yt/src/commands/subscriptions/mod.rs new file mode 100644 index 0000000..6a16a9a --- /dev/null +++ b/crates/yt/src/commands/subscriptions/mod.rs @@ -0,0 +1,84 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::PathBuf; + +use clap::{Subcommand, ValueEnum}; +use clap_complete::ArgValueCompleter; +use url::Url; + +use crate::commands::complete_subscription; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +pub(super) enum SubscriptionCommand { + /// Subscribe to an URL + Add { + #[arg(short, long)] + /// The human readable name of the subscription + name: Option<String>, + + /// The URL to listen to + url: Url, + + /// Don't check, whether the URL actually points to something yt understands. + #[arg(long, default_value_t = false)] + no_check: bool, + }, + + /// Unsubscribe from an URL + Remove { + /// The human readable name of the subscription + #[arg(add = ArgValueCompleter::new(complete_subscription))] + name: String, + }, + + /// Change the status of an subscription. + /// + /// An active subscription will be updated in `yt update`, while an inactive one will not. + SetStatus { + /// The human readable name of the subscription + #[arg(add = ArgValueCompleter::new(complete_subscription))] + name: String, + + /// What should this subscription be considered now? + new_status: SubscriptionStatus, + }, + + /// Import a bunch of URLs as subscriptions. + Import { + /// The file containing the URLs. Will use Stdin otherwise. + file: Option<PathBuf>, + + /// Remove any previous subscriptions + #[arg(short, long)] + force: bool, + + /// Don't check, whether the URLs actually point to something yt understands. + #[arg(long, default_value_t = false)] + no_check: bool, + }, + /// Write all subscriptions in an format understood by `import` + Export {}, + + /// List all subscriptions + List { + /// Only show active subscriptions + #[arg(short, long, default_value_t = false)] + active: bool, + }, +} + +#[derive(ValueEnum, Debug, Clone, Copy)] +pub(in crate::commands) enum SubscriptionStatus { + Active, + Inactive, +} diff --git a/crates/yt/src/commands/update/implm/mod.rs b/crates/yt/src/commands/update/implm/mod.rs new file mode 100644 index 0000000..10626ac --- /dev/null +++ b/crates/yt/src/commands/update/implm/mod.rs @@ -0,0 +1,62 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::update::{UpdateCommand, implm::updater::Updater}, + storage::db::{ + extractor_hash::ExtractorHash, + subscription::{Subscription, Subscriptions}, + }, +}; + +use anyhow::{Result, bail}; + +mod updater; + +impl UpdateCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let UpdateCommand { + max_backlog, + subscriptions: subscription_names_to_update, + } = self; + + let mut all_subs = Subscriptions::get(app).await?.remove_inactive(); + + let max_backlog = max_backlog.unwrap_or(app.config.update.max_backlog); + + let subs: Vec<Subscription> = if subscription_names_to_update.is_empty() { + all_subs.0.into_values().collect() + } else { + subscription_names_to_update + .into_iter() + .map(|sub| { + if let Some(val) = all_subs.0.remove(&sub) { + Ok(val) + } else { + bail!( + "Your specified subscription to update '{}' is not a subscription!", + sub + ) + } + }) + .collect::<Result<_>>()? + }; + + // We can get away with not having to re-fetch the hashes every time, as the returned video + // should not contain duplicates. + let hashes = ExtractorHash::get_all(app).await?; + + let updater = Updater::new(max_backlog, app.config.update.pool_size, hashes); + updater.update(app, subs).await?; + + Ok(()) + } +} diff --git a/crates/yt/src/commands/update/implm/updater.rs b/crates/yt/src/commands/update/implm/updater.rs new file mode 100644 index 0000000..2b96bf2 --- /dev/null +++ b/crates/yt/src/commands/update/implm/updater.rs @@ -0,0 +1,205 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use anyhow::{Context, Result}; +use futures::{StreamExt, future::join_all, stream}; +use log::{Level, debug, error, log_enabled}; +use tokio::io::{AsyncWriteExt, stderr}; +use tokio_util::task::LocalPoolHandle; +use yt_dlp::{ + info_json::InfoJson, json_cast, json_try_get, options::YoutubeDLOptions, process_ie_result, + python_error::PythonError, +}; + +use crate::{ + ansi_escape_codes, + app::App, + storage::db::{ + extractor_hash::ExtractorHash, insert::Operations, subscription::Subscription, video::Video, + }, + yt_dlp::yt_dlp_opts_updating, +}; + +pub(super) struct Updater { + max_backlog: usize, + hashes: Vec<ExtractorHash>, + pool: LocalPoolHandle, +} + +static REACHED_NUMBER: AtomicUsize = const { AtomicUsize::new(1) }; + +impl Updater { + pub(super) fn new(max_backlog: usize, max_threads: usize, hashes: Vec<ExtractorHash>) -> Self { + let pool = LocalPoolHandle::new(max_threads); + + Self { + max_backlog, + hashes, + pool, + } + } + + pub(super) async fn update(self, app: &App, subscriptions: Vec<Subscription>) -> Result<()> { + let total_number = subscriptions.len(); + + let mut stream = stream::iter(subscriptions) + .map(|sub| self.get_new_entries(sub, total_number)) + .buffer_unordered(app.config.update.futures); + + while let Some(output) = stream.next().await { + let mut entries = output?; + + if let Some(next) = entries.next() { + let (sub, entry) = next; + process_subscription(app, sub, entry).await?; + + join_all(entries.map(|(sub, entry)| process_subscription(app, sub, entry))) + .await + .into_iter() + .collect::<Result<(), _>>()?; + } + } + + Ok(()) + } + + #[allow(clippy::too_many_lines)] + async fn get_new_entries( + &self, + sub: Subscription, + total_number: usize, + ) -> Result<impl Iterator<Item = (Subscription, InfoJson)>> { + let max_backlog = self.max_backlog; + let hashes = self.hashes.clone(); + + let yt_dlp = yt_dlp_opts_updating(max_backlog)?; + + self.pool + .spawn_pinned(move || { + async move { + if !log_enabled!(Level::Debug) { + ansi_escape_codes::clear_whole_line(); + ansi_escape_codes::move_to_col(1); + eprint!( + "({}/{total_number}) Checking playlist {}...", + REACHED_NUMBER.fetch_add(1, Ordering::Relaxed), + sub.name + ); + ansi_escape_codes::move_to_col(1); + stderr().flush().await?; + } + + let info = yt_dlp + .extract_info(&sub.url, false, false) + .with_context(|| format!("Failed to get playlist '{}'.", sub.name))?; + + let empty = vec![]; + let entries = json_try_get!(info, "entries", as_array).unwrap_or(&empty); + + let valid_entries: Vec<(Subscription, InfoJson)> = entries + .iter() + .take(max_backlog) + .filter_map(|entry| -> Option<(Subscription, InfoJson)> { + let extractor_hash = + ExtractorHash::from_info_json(json_cast!(entry, as_object)); + + if hashes.contains(&extractor_hash) { + debug!( + "Skipping entry, as it is \ + already present: '{extractor_hash}'", + ); + None + } else { + Some((sub.clone(), json_cast!(entry, as_object).to_owned())) + } + }) + .collect(); + + Ok(valid_entries + .into_iter() + .map(|(sub, entry)| { + let inner_yt_dlp = YoutubeDLOptions::new() + .set("noplaylist", true) + .build() + .expect("Worked before, should work now"); + + match inner_yt_dlp.process_ie_result(entry, false) { + Ok(output) => Ok((sub, output)), + Err(err) => Err(err), + } + }) + // Don't fail the whole update, if one of the entries fails to fetch. + .filter_map(move |base| match base { + Ok(ok) => Some(ok), + Err(err) => { + match err { + process_ie_result::Error::Python(PythonError(err)) => { + if err.contains( + "Join this channel to get access \ + to members-only content ", + ) { + // Hide this error + } else { + // Show the error, but don't fail. + let error = err + .strip_prefix( + "DownloadError: \u{1b}[0;31mERROR:\u{1b}[0m ", + ) + .unwrap_or(&err); + error!("While fetching {:#?}: {error}", sub.name); + } + + None + } + process_ie_result::Error::InfoJsonPrepare(error) => { + error!( + "While fetching {:#?}: Failed to prepare \ + info json: {error}", + sub.name + ); + None + } + } + } + })) + } + }) + .await? + } +} + +async fn process_subscription(app: &App, sub: Subscription, entry: InfoJson) -> Result<()> { + let mut ops = Operations::new("Update: process subscription"); + let video = Video::from_info_json(&entry, Some(&sub)) + .context("Failed to parse search entry as Video")?; + + let title = video.title.clone(); + let url = video.url.clone(); + let video = video.add(&mut ops).with_context(|| { + format!("Failed to add video to database: '{title}' (with url: '{url}')") + })?; + + ops.commit(app).await.with_context(|| { + format!( + "Failed to add video to database: '{}' (with url: '{}')", + video.title, video.url + ) + })?; + println!( + "{}", + &video + .to_line_display(app, None) + .await + .with_context(|| format!("Failed to format video: '{}'", video.title))? + ); + Ok(()) +} diff --git a/crates/yt/src/commands/update/mod.rs b/crates/yt/src/commands/update/mod.rs new file mode 100644 index 0000000..cb29148 --- /dev/null +++ b/crates/yt/src/commands/update/mod.rs @@ -0,0 +1,27 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; +use clap_complete::ArgValueCompleter; + +use crate::commands::complete_subscription; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct UpdateCommand { + /// The maximal number of videos to fetch for each subscription. + #[arg(short, long)] + max_backlog: Option<usize>, + + /// The subscriptions to update + #[arg(add = ArgValueCompleter::new(complete_subscription))] + subscriptions: Vec<String>, +} diff --git a/crates/yt/src/commands/videos/implm.rs b/crates/yt/src/commands/videos/implm.rs new file mode 100644 index 0000000..2a018c7 --- /dev/null +++ b/crates/yt/src/commands/videos/implm.rs @@ -0,0 +1,73 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::videos::VideosCommand, + storage::db::video::{Video, VideoStatusMarker}, +}; + +use anyhow::{Context, Result}; +use futures::{TryStreamExt, stream::FuturesUnordered}; + +impl VideosCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + VideosCommand::List { + search_query, + limit, + format, + } => { + let all_videos = Video::in_states(app, VideoStatusMarker::ALL).await?; + + // turn one video to a color display, to pre-warm the hash shrinking cache + if let Some(val) = all_videos.first() { + val.to_line_display(app, format.clone()).await?; + } + + let limit = limit.unwrap_or(all_videos.len()); + + let all_video_strings: Vec<String> = all_videos + .into_iter() + .take(limit) + .map(|vid| to_line_display_owned(vid, app, format.clone())) + .collect::<FuturesUnordered<_>>() + .try_collect::<Vec<String>>() + .await?; + + if let Some(query) = search_query { + all_video_strings + .into_iter() + .filter(|video| video.to_lowercase().contains(&query.to_lowercase())) + .for_each(|video| println!("{video}")); + } else { + println!("{}", all_video_strings.join("\n")); + } + } + VideosCommand::Info { hash, format } => { + let video = hash.realize(app, None).await?.get_with_app(app).await?; + + print!( + "{}", + &video + .to_info_display(app, format) + .await + .context("Failed to format video")? + ); + } + } + + Ok(()) + } +} + +async fn to_line_display_owned(video: Video, app: &App, format: Option<String>) -> Result<String> { + video.to_line_display(app, format).await +} diff --git a/crates/yt/src/commands/videos/mod.rs b/crates/yt/src/commands/videos/mod.rs new file mode 100644 index 0000000..ca20715 --- /dev/null +++ b/crates/yt/src/commands/videos/mod.rs @@ -0,0 +1,46 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::{ArgAction, Subcommand}; + +use crate::storage::db::extractor_hash::LazyExtractorHash; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +pub(super) enum VideosCommand { + /// List the videos in the database + #[command(visible_alias = "ls")] + List { + /// An optional search query to limit the results + #[arg(action = ArgAction::Append)] + search_query: Option<String>, + + /// The format string to use. + // TODO(@bpeetz): Encode the default format, as the default string here. <2025-07-04> + #[arg(short, long)] + format: Option<String>, + + /// The number of videos to show + #[arg(short, long)] + limit: Option<usize>, + }, + + /// Get detailed information about a video + Info { + /// The short hash of the video + hash: LazyExtractorHash, + + /// The format string to use. + // TODO(@bpeetz): Encode the default format, as the default string here. <2025-07-04> + #[arg(short, long)] + format: Option<String>, + }, +} diff --git a/crates/yt/src/commands/watch/implm/mod.rs b/crates/yt/src/commands/watch/implm/mod.rs new file mode 100644 index 0000000..8182216 --- /dev/null +++ b/crates/yt/src/commands/watch/implm/mod.rs @@ -0,0 +1,244 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fs, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use crate::{ + app::App, + commands::watch::{WatchCommand, implm::playlist_handler::Status}, + storage::{ + db::{ + insert::{Operations, maintenance::clear_stale_downloaded_paths}, + playlist::Playlist, + }, + notify::wait_for_db_write, + }, +}; + +use anyhow::{Context, Result}; +use libmpv2::{Mpv, events::EventContext}; +use log::{debug, info, trace, warn}; +use tokio::{task, time}; + +mod playlist_handler; + +impl WatchCommand { + #[allow(clippy::too_many_lines)] + pub(in crate::commands) async fn implm(self, app: Arc<App>) -> Result<()> { + let WatchCommand { + provide_ipc_socket, + headless, + } = self; + + clear_stale_downloaded_paths(&app).await?; + + let ipc_socket = if provide_ipc_socket { + Some(app.config.paths.mpv_ipc_socket_path.clone()) + } else { + None + }; + + let (mpv, mut ev_ctx) = + init_mpv(&app, ipc_socket, headless).context("Failed to initialize mpv instance")?; + let mpv = Arc::new(mpv); + + if provide_ipc_socket { + println!("{}", app.config.paths.mpv_ipc_socket_path.display()); + } + + let should_break = Arc::new(AtomicBool::new(false)); + let local_app = Arc::clone(&app); + let local_mpv = Arc::clone(&mpv); + let local_should_break = Arc::clone(&should_break); + let progress_handle = task::spawn(async move { + loop { + if local_should_break.load(Ordering::Relaxed) { + trace!("WatchProgressThread: Stopping, as we received exit signal."); + break; + } + + let mut playlist = Playlist::create(&local_app).await?; + + if let Some(index) = playlist.current_index() { + trace!("WatchProgressThread: Saving watch progress for current video"); + + let mut ops = + Operations::new("WatchProgressThread: save watch progress thread"); + playlist.save_watch_progress(&local_mpv, index, &mut ops); + ops.commit(&local_app).await?; + } else { + trace!( + "WatchProgressThread: Tried to save current watch progress, but no video active." + ); + } + + time::sleep(local_app.config.watch.progress_save_intervall).await; + } + + Ok::<(), anyhow::Error>(()) + }); + + let playlist = Playlist::create(&app).await?; + playlist.resync_with_mpv(&app, &mpv)?; + + let mut have_warned = (false, 0); + 'watchloop: loop { + 'waitloop: while let Ok(value) = playlist_handler::status(&app).await { + match value { + Status::NoMoreAvailable => { + break 'watchloop; + } + Status::NoCached { marked_watch } => { + // try again next time. + if have_warned.0 { + if have_warned.1 != marked_watch { + warn!("Now {marked_watch} videos are marked as to be watched."); + have_warned.1 = marked_watch; + } + } else { + warn!( + "There is nothing to watch yet, but still {marked_watch} videos marked as to be watched. \ + Will idle, until they become available" + ); + have_warned = (true, marked_watch); + } + wait_for_db_write(&app).await?; + + // Add the new videos, if they are there. + let playlist = Playlist::create(&app).await?; + playlist.resync_with_mpv(&app, &mpv)?; + } + Status::Available { newly_available } => { + debug!( + "Checked for currently available videos and found {newly_available}!" + ); + have_warned.0 = false; + + // Something just became available! + break 'waitloop; + } + } + } + + // TODO(@bpeetz): Is the following assumption correct? <2025-07-10> + // We wait until forever for the next event, because we really don't need to do anything + // else. + if let Some(ev) = ev_ctx.wait_event(f64::MAX) { + match ev { + Ok(event) => { + trace!("Mpv event triggered: {event:#?}"); + if playlist_handler::handle_mpv_event(&app, &mpv, &event) + .await + .with_context(|| format!("Failed to handle mpv event: '{event:#?}'"))? + { + break; + } + } + Err(e) => debug!("Mpv Event errored: {e}"), + } + } + } + should_break.store(true, Ordering::Relaxed); + progress_handle.await??; + + if provide_ipc_socket { + fs::remove_file(&app.config.paths.mpv_ipc_socket_path).with_context(|| { + format!( + "Failed to clean-up the mpv ipc socket at {}", + app.config.paths.mpv_ipc_socket_path.display() + ) + })?; + } + + Ok(()) + } +} + +fn init_mpv(app: &App, ipc_socket: Option<PathBuf>, headless: bool) -> Result<(Mpv, EventContext)> { + // set some default values, to make things easier (these can be overridden by the config file, + // which we load later) + let mpv = Mpv::with_initializer(|mpv| { + if let Some(socket) = ipc_socket { + mpv.set_property( + "input-ipc-server", + socket + .to_str() + .expect("This path comes from us, it should never contain not-utf8"), + )?; + } + + if headless { + // Do not provide video output. + mpv.set_property("vid", "no")?; + } else { + // Enable default key bindings, so the user can actually interact with + // the player (and e.g. close the window). + mpv.set_property("input-default-bindings", "yes")?; + mpv.set_property("input-vo-keyboard", "yes")?; + + // Show the on screen controller. + mpv.set_property("osc", "yes")?; + + // Don't automatically advance to the next video (or exit the player) + mpv.set_option("keep-open", "always")?; + + // Always display an window, even for non-video playback. + // As mpv does not have cli access, no window means no control and no user feedback. + mpv.set_option("force-window", "yes")?; + } + + Ok(()) + }) + .context("Failed to initialize mpv")?; + + let config_path = &app.config.paths.mpv_config_path; + if config_path.try_exists()? { + info!("Found mpv.conf at '{}'!", config_path.display()); + mpv.command( + "load-config-file", + &[config_path + .to_str() + .context("Failed to parse the config path is utf8-stringt")?], + )?; + } else { + warn!( + "Did not find a mpv.conf file at '{}'", + config_path.display() + ); + } + + let input_path = &app.config.paths.mpv_input_path; + if input_path.try_exists()? { + info!("Found mpv.input.conf at '{}'!", input_path.display()); + mpv.command( + "load-input-conf", + &[input_path + .to_str() + .context("Failed to parse the input path as utf8 string")?], + )?; + } else { + warn!( + "Did not find a mpv.input.conf file at '{}'", + input_path.display() + ); + } + + let ev_ctx = EventContext::new(mpv.ctx); + ev_ctx.disable_deprecated_events()?; + + Ok((mpv, ev_ctx)) +} diff --git a/crates/yt/src/watch/playlist_handler/client_messages/mod.rs b/crates/yt/src/commands/watch/implm/playlist_handler/client_messages.rs index c05ca87..fd7e035 100644 --- a/crates/yt/src/watch/playlist_handler/client_messages/mod.rs +++ b/crates/yt/src/commands/watch/implm/playlist_handler/client_messages.rs @@ -10,7 +10,7 @@ use std::{env, time::Duration}; -use crate::{app::App, comments}; +use crate::{app::App, storage::db::video::Video}; use anyhow::{Context, Result, bail}; use libmpv2::Mpv; @@ -23,19 +23,8 @@ async fn run_self_in_external_command(app: &App, args: &[&str]) -> Result<()> { let binary = env::current_exe().context("Failed to determine the current executable to re-execute")?; - let status = Command::new("riverctl") - .args(["focus-output", "next"]) - .status() - .await?; - if !status.success() { - bail!("focusing the next output failed!"); - } - let arguments = [ &[ - "--title", - "floating please", - "--command", binary .to_str() .context("Failed to turn the executable path to a utf8-string")?, @@ -50,29 +39,24 @@ async fn run_self_in_external_command(app: &App, args: &[&str]) -> Result<()> { ] .concat(); - let status = Command::new("alacritty").args(arguments).status().await?; - if !status.success() { - bail!("Falied to start `yt comments`"); - } - - let status = Command::new("riverctl") - .args(["focus-output", "next"]) + let status = Command::new(app.config.commands.external_spawn.first()) + .args(app.config.commands.external_spawn.tail()) + .args(arguments) .status() .await?; - if !status.success() { - bail!("focusing the next output failed!"); + bail!("Falied to start (external) `yt {}`", args.join(" ")); } Ok(()) } pub(super) async fn handle_yt_description_external(app: &App) -> Result<()> { - run_self_in_external_command(app, &["description"]).await?; + run_self_in_external_command(app, &["show", "description"]).await?; Ok(()) } pub(super) async fn handle_yt_description_local(app: &App, mpv: &Mpv) -> Result<()> { - let description: String = comments::description::get(app) + let description: String = Video::get_current_description(app) .await? .chars() .take(app.config.watch.local_displays_length) @@ -83,11 +67,11 @@ pub(super) async fn handle_yt_description_local(app: &App, mpv: &Mpv) -> Result< } pub(super) async fn handle_yt_comments_external(app: &App) -> Result<()> { - run_self_in_external_command(app, &["comments"]).await?; + run_self_in_external_command(app, &["show", "comments"]).await?; Ok(()) } pub(super) async fn handle_yt_comments_local(app: &App, mpv: &Mpv) -> Result<()> { - let comments: String = comments::get(app) + let comments: String = Video::get_current_comments(app) .await? .render(false) .chars() @@ -97,3 +81,13 @@ pub(super) async fn handle_yt_comments_local(app: &App, mpv: &Mpv) -> Result<()> mpv_message(mpv, &comments, Duration::from_secs(6))?; Ok(()) } + +pub(super) async fn handle_yt_info_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "info"]).await?; + Ok(()) +} + +pub(super) async fn handle_yt_thumbnail_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "thumbnail"]).await?; + Ok(()) +} diff --git a/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs b/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs new file mode 100644 index 0000000..bdb77d2 --- /dev/null +++ b/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs @@ -0,0 +1,225 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::time::Duration; + +use crate::{ + app::App, + storage::db::{ + insert::{Operations, playlist::VideoTransition}, + playlist::{Playlist, PlaylistIndex}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::{Context, Result}; +use libmpv2::{EndFileReason, Mpv, events::Event}; +use log::{debug, info}; + +mod client_messages; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum Status { + /// There are no videos cached and no more marked to be watched. + /// Waiting is pointless. + NoMoreAvailable, + + /// There are no videos cached, but some (> 0) are marked to be watched. + /// So we should wait for them to become available. + NoCached { marked_watch: usize }, + + /// There are videos cached and ready to be inserted into the playback queue. + Available { newly_available: usize }, +} + +fn mpv_message(mpv: &Mpv, message: &str, time: Duration) -> Result<()> { + mpv.command( + "show-text", + &[message, time.as_millis().to_string().as_str()], + )?; + Ok(()) +} + +/// Return the status of the playback queue +pub(crate) async fn status(app: &App) -> Result<Status> { + let playlist = Playlist::create(app).await?; + + let playlist_len = playlist.len(); + let marked_watch_num = Video::in_states(app, &[VideoStatusMarker::Watch]) + .await? + .len(); + + if playlist_len == 0 && marked_watch_num == 0 { + Ok(Status::NoMoreAvailable) + } else if playlist_len == 0 && marked_watch_num != 0 { + Ok(Status::NoCached { + marked_watch: marked_watch_num, + }) + } else if playlist_len != 0 { + Ok(Status::Available { + newly_available: playlist_len, + }) + } else { + unreachable!( + "The playlist length is {playlist_len}, but the number of marked watch videos is {marked_watch_num}! This is a bug." + ); + } +} + +/// # Returns +/// This will return [`true`], if the event handling should be stopped +/// +/// # Panics +/// Only if internal assertions fail. +#[allow(clippy::too_many_lines)] +pub(crate) async fn handle_mpv_event(app: &App, mpv: &Mpv, event: &Event<'_>) -> Result<bool> { + let mut ops = Operations::new("PlaylistHandler: handle event"); + + // Construct the playlist lazily. + // This avoids unneeded db lookups. + // (We use the moved `call_once` as guard for this) + let call_once = String::new(); + let playlist = move || { + drop(call_once); + Playlist::create(app) + }; + + let should_stop_event_handling = match event { + Event::EndFile(r) => match r.reason { + EndFileReason::Eof => { + info!("Mpv reached the end of the current video. Marking it watched."); + playlist().await?.resync_with_mpv(app, mpv)?; + + false + } + EndFileReason::Stop => { + // This reason is incredibly ambiguous. It _both_ means actually pausing a + // video and going to the next one in the playlist. + // Oh, and it's also called, when a video is removed from the playlist (at + // least via "playlist-remove current") + info!("Paused video (or went to next playlist entry); Doing nothing"); + + false + } + EndFileReason::Quit => { + info!("Mpv quit. Exiting playback"); + + playlist().await?.save_current_watch_progress(mpv, &mut ops); + + true + } + EndFileReason::Error => { + unreachable!("This should have been raised as a separate error") + } + EndFileReason::Redirect => { + // TODO: We probably need to handle this somehow <2025-02-17> + false + } + }, + Event::StartFile(_) => { + let mut playlist = playlist().await?; + + let mpv_pos = usize::try_from(mpv.get_property::<i64>("playlist-pos")?) + .expect("The value is strictly positive"); + + let yt_pos = playlist.current_index().map(usize::from); + + if (Some(mpv_pos) != yt_pos) || yt_pos.is_none() { + debug!( + "StartFileHandler: mpv pos {mpv_pos} and our pos {yt_pos:?} do not align. Reloading.." + ); + + if let Some((_, vid)) = playlist.get_focused_mut() { + vid.set_focused(false, &mut ops); + ops.commit(app) + .await + .context("Failed to commit video unfocusing")?; + + ops = Operations::new("PlaylistHandler: after set-focused"); + } + + let video = playlist + .get_mut(PlaylistIndex::from(mpv_pos)) + .expect("The mpv pos should not be out of bounds"); + + video.set_focused(true, &mut ops); + + playlist.resync_with_mpv(app, mpv)?; + } + + false + } + Event::Seek => { + playlist().await?.save_current_watch_progress(mpv, &mut ops); + + false + } + Event::ClientMessage(a) => { + debug!("Got Client Message event: '{}'", a.join(" ")); + + match a.as_slice() { + &["yt-comments-external"] => { + client_messages::handle_yt_comments_external(app).await?; + } + &["yt-comments-local"] => { + client_messages::handle_yt_comments_local(app, mpv).await?; + } + + &["yt-description-external"] => { + client_messages::handle_yt_description_external(app).await?; + } + &["yt-description-local"] => { + client_messages::handle_yt_description_local(app, mpv).await?; + } + + &["yt-info-external"] => { + client_messages::handle_yt_info_external(app).await?; + } + &["yt-thumbnail-external"] => { + client_messages::handle_yt_thumbnail_external(app).await?; + } + + &["yt-mark-picked"] => { + playlist().await?.mark_current_done( + app, + mpv, + VideoTransition::Picked, + &mut ops, + )?; + + mpv_message(mpv, "Marked the video as picked", Duration::from_secs(3))?; + } + &["yt-mark-watched"] => { + playlist().await?.mark_current_done( + app, + mpv, + VideoTransition::Watched, + &mut ops, + )?; + + mpv_message(mpv, "Marked the video watched", Duration::from_secs(3))?; + } + &["yt-check-new-videos"] => { + playlist().await?.resync_with_mpv(app, mpv)?; + } + other => { + debug!("Unknown message: {}", other.join(" ")); + } + } + + false + } + _ => false, + }; + + ops.commit(app).await?; + + Ok(should_stop_event_handling) +} diff --git a/crates/yt/src/commands/watch/mod.rs b/crates/yt/src/commands/watch/mod.rs new file mode 100644 index 0000000..ea4c513 --- /dev/null +++ b/crates/yt/src/commands/watch/mod.rs @@ -0,0 +1,24 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct WatchCommand { + /// Print the path to an ipc socket for mpv control to stdout at startup. + #[arg(long)] + provide_ipc_socket: bool, + + /// Don't start an mpv window at all. + #[arg(long)] + headless: bool, +} diff --git a/crates/yt/src/comments/comment.rs b/crates/yt/src/comments/comment.rs deleted file mode 100644 index 5bc939c..0000000 --- a/crates/yt/src/comments/comment.rs +++ /dev/null @@ -1,152 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use serde::{Deserialize, Deserializer, Serialize}; -use url::Url; - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(from = "String")] -#[serde(deny_unknown_fields)] -pub enum Parent { - Root, - Id(String), -} - -impl Parent { - #[must_use] - pub fn id(&self) -> Option<&str> { - if let Self::Id(id) = self { - Some(id) - } else { - None - } - } -} - -impl From<String> for Parent { - fn from(value: String) -> Self { - if value == "root" { - Self::Root - } else { - Self::Id(value) - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(from = "String")] -#[serde(deny_unknown_fields)] -pub struct Id { - pub id: String, -} -impl From<String> for Id { - fn from(value: String) -> Self { - Self { - // Take the last element if the string is split with dots, otherwise take the full id - id: value.split('.').last().unwrap_or(&value).to_owned(), - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[allow(clippy::struct_excessive_bools)] -pub struct Comment { - pub id: Id, - pub text: String, - #[serde(default = "zero")] - pub like_count: u32, - pub is_pinned: bool, - pub author_id: String, - #[serde(default = "unknown")] - pub author: String, - pub author_is_verified: bool, - pub author_thumbnail: Url, - pub parent: Parent, - #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] - pub edited: bool, - // Can't also be deserialized, as it's already used in 'edited' - // _time_text: String, - pub timestamp: i64, - pub author_url: Option<Url>, - pub author_is_uploader: bool, - pub is_favorited: bool, -} - -fn unknown() -> String { - "<Unknown>".to_string() -} -fn zero() -> u32 { - 0 -} -fn edited_from_time_text<'de, D>(d: D) -> Result<bool, D::Error> -where - D: Deserializer<'de>, -{ - let s = String::deserialize(d)?; - if s.contains(" (edited)") { - Ok(true) - } else { - Ok(false) - } -} - -#[derive(Debug, Clone)] -#[allow(clippy::module_name_repetitions)] -pub struct CommentExt { - pub value: Comment, - pub replies: Vec<CommentExt>, -} - -#[derive(Debug, Default)] -pub struct Comments { - pub(super) vec: Vec<CommentExt>, -} - -impl Comments { - pub fn new() -> Self { - Self::default() - } - pub fn push(&mut self, value: CommentExt) { - self.vec.push(value); - } - pub fn get_mut(&mut self, key: &str) -> Option<&mut CommentExt> { - self.vec.iter_mut().filter(|c| c.value.id.id == key).last() - } - pub fn insert(&mut self, key: &str, value: CommentExt) { - let parent = self - .vec - .iter_mut() - .filter(|c| c.value.id.id == key) - .last() - .expect("One of these should exist"); - parent.push_reply(value); - } -} -impl CommentExt { - pub fn push_reply(&mut self, value: CommentExt) { - self.replies.push(value); - } - pub fn get_mut_reply(&mut self, key: &str) -> Option<&mut CommentExt> { - self.replies - .iter_mut() - .filter(|c| c.value.id.id == key) - .last() - } -} - -impl From<Comment> for CommentExt { - fn from(value: Comment) -> Self { - Self { - replies: vec![], - value, - } - } -} diff --git a/crates/yt/src/comments/description.rs b/crates/yt/src/comments/description.rs deleted file mode 100644 index e8cb29d..0000000 --- a/crates/yt/src/comments/description.rs +++ /dev/null @@ -1,46 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use crate::{ - App, - comments::output::display_fmt_and_less, - storage::video_database::{Video, get}, - unreachable::Unreachable, -}; - -use anyhow::{Result, bail}; -use yt_dlp::{InfoJson, json_cast}; - -pub async fn description(app: &App) -> Result<()> { - let description = get(app).await?; - display_fmt_and_less(description).await?; - - Ok(()) -} - -pub async fn get(app: &App) -> Result<String> { - let currently_playing_video: Video = - if let Some(video) = get::currently_focused_video(app).await? { - video - } else { - bail!("Could not find a currently playing video!"); - }; - - let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( - "A currently *playing* must be cached. And thus the info.json should be available", - ); - - Ok(info_json - .get("description") - .map(|val| json_cast!(val, as_str)) - .unwrap_or("<No description>") - .to_owned()) -} diff --git a/crates/yt/src/comments/mod.rs b/crates/yt/src/comments/mod.rs deleted file mode 100644 index 876146d..0000000 --- a/crates/yt/src/comments/mod.rs +++ /dev/null @@ -1,167 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::mem; - -use anyhow::{Result, bail}; -use comment::{Comment, CommentExt, Comments, Parent}; -use output::display_fmt_and_less; -use regex::Regex; -use yt_dlp::{InfoJson, json_cast}; - -use crate::{ - app::App, - storage::video_database::{Video, get}, - unreachable::Unreachable, -}; - -mod comment; -mod display; -pub mod output; - -pub mod description; -pub use description::*; - -#[allow(clippy::too_many_lines)] -pub async fn get(app: &App) -> Result<Comments> { - let currently_playing_video: Video = - if let Some(video) = get::currently_focused_video(app).await? { - video - } else { - bail!("Could not find a currently playing video!"); - }; - - let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( - "A currently *playing* video must be cached. And thus the info.json should be available", - ); - - let base_comments = if let Some(comments) = info_json.get("comments") { - json_cast!(comments, as_array) - } else { - bail!( - "The video ('{}') does not have comments!", - info_json - .get("title") - .map(|val| json_cast!(val, as_str)) - .unwrap_or("<No Title>") - ) - }; - - let mut comments = Comments::new(); - for c in base_comments { - let c: Comment = serde_json::from_value(c.to_owned())?; - if let Parent::Id(id) = &c.parent { - comments.insert(&(id.clone()), CommentExt::from(c)); - } else { - comments.push(CommentExt::from(c)); - } - } - - comments.vec.iter_mut().for_each(|comment| { - let replies = mem::take(&mut comment.replies); - let mut output_replies: Vec<CommentExt> = vec![]; - - let re = Regex::new(r"\u{200b}?(@[^\t\s]+)\u{200b}?").unreachable("This is hardcoded"); - for reply in replies { - if let Some(replyee_match) = re.captures(&reply.value.text){ - let full_match = replyee_match.get(0).unreachable("This will always exist"); - let text = reply. - value. - text[0..full_match.start()] - .to_owned() - + - &reply - .value - .text[full_match.end()..]; - let text: &str = text.trim().trim_matches('\u{200b}'); - - let replyee = replyee_match.get(1).unreachable("This should also exist").as_str(); - - - if let Some(parent) = output_replies - .iter_mut() - // .rev() - .flat_map(|com| &mut com.replies) - .flat_map(|com| &mut com.replies) - .flat_map(|com| &mut com.replies) - .filter(|com| com.value.author == replyee) - .last() - { - parent.replies.push(CommentExt::from(Comment { - text: text.to_owned(), - ..reply.value - })); - } else if let Some(parent) = output_replies - .iter_mut() - // .rev() - .flat_map(|com| &mut com.replies) - .flat_map(|com| &mut com.replies) - .filter(|com| com.value.author == replyee) - .last() - { - parent.replies.push(CommentExt::from(Comment { - text: text.to_owned(), - ..reply.value - })); - } else if let Some(parent) = output_replies - .iter_mut() - // .rev() - .flat_map(|com| &mut com.replies) - .filter(|com| com.value.author == replyee) - .last() - { - parent.replies.push(CommentExt::from(Comment { - text: text.to_owned(), - ..reply.value - })); - } else if let Some(parent) = output_replies.iter_mut() - // .rev() - .filter(|com| com.value.author == replyee) - .last() - { - parent.replies.push(CommentExt::from(Comment { - text: text.to_owned(), - ..reply.value - })); - } else { - eprintln!( - "Failed to find a parent for ('{}') both directly and via replies! The reply text was:\n'{}'\n", - replyee, - reply.value.text - ); - output_replies.push(reply); - } - } else { - output_replies.push(reply); - } - } - comment.replies = output_replies; - }); - - Ok(comments) -} - -pub async fn comments(app: &App) -> Result<()> { - let comments = get(app).await?; - - display_fmt_and_less(comments.render(true)).await?; - - Ok(()) -} - -#[cfg(test)] -mod test { - #[test] - fn test_string_replacement() { - let s = "A \n\nB\n\nC".to_owned(); - assert_eq!("A \n \n B\n \n C", s.replace('\n', "\n ")); - } -} diff --git a/crates/yt/src/config/default.rs b/crates/yt/src/config/default.rs deleted file mode 100644 index 4ed643b..0000000 --- a/crates/yt/src/config/default.rs +++ /dev/null @@ -1,110 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::path::PathBuf; - -use anyhow::{Context, Result}; - -fn get_runtime_path(name: &'static str) -> Result<PathBuf> { - let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); - xdg_dirs - .place_runtime_file(name) - .with_context(|| format!("Failed to place runtime file: '{name}'")) -} -fn get_data_path(name: &'static str) -> Result<PathBuf> { - let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); - xdg_dirs - .place_data_file(name) - .with_context(|| format!("Failed to place data file: '{name}'")) -} -fn get_config_path(name: &'static str) -> Result<PathBuf> { - let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); - xdg_dirs - .place_config_file(name) - .with_context(|| format!("Failed to place config file: '{name}'")) -} - -pub(super) fn create_path(path: PathBuf) -> Result<PathBuf> { - if !path.exists() { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; - } - } - - Ok(path) -} - -pub(crate) const PREFIX: &str = "yt"; - -pub(crate) mod global { - pub(crate) fn display_colors() -> bool { - // TODO: This should probably check if the output is a tty and otherwise return `false` <2025-02-14> - true - } -} - -pub(crate) mod select { - pub(crate) fn playback_speed() -> f64 { - 2.7 - } - pub(crate) fn subtitle_langs() -> &'static str { - "" - } -} - -pub(crate) mod watch { - pub(crate) fn local_displays_length() -> usize { - 1000 - } -} - -pub(crate) mod update { - pub(crate) fn max_backlog() -> usize { - 20 - } -} - -pub(crate) mod paths { - use std::{env::temp_dir, path::PathBuf}; - - use anyhow::Result; - - use super::{PREFIX, create_path, get_config_path, get_data_path, get_runtime_path}; - - // We download to the temp dir to avoid taxing the disk - pub(crate) fn download_dir() -> Result<PathBuf> { - let temp_dir = temp_dir(); - - create_path(temp_dir.join(PREFIX)) - } - pub(crate) fn mpv_config_path() -> Result<PathBuf> { - get_config_path("mpv.conf") - } - pub(crate) fn mpv_input_path() -> Result<PathBuf> { - get_config_path("mpv.input.conf") - } - pub(crate) fn database_path() -> Result<PathBuf> { - get_data_path("videos.sqlite") - } - pub(crate) fn config_path() -> Result<PathBuf> { - get_config_path("config.toml") - } - pub(crate) fn last_selection_path() -> Result<PathBuf> { - get_runtime_path("selected.yts") - } -} - -pub(crate) mod download { - pub(crate) fn max_cache_size() -> &'static str { - "3 GiB" - } -} diff --git a/crates/yt/src/config/definitions.rs b/crates/yt/src/config/definitions.rs deleted file mode 100644 index ce8c0d4..0000000 --- a/crates/yt/src/config/definitions.rs +++ /dev/null @@ -1,67 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::path::PathBuf; - -use serde::Deserialize; - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub(crate) struct ConfigFile { - pub global: Option<GlobalConfig>, - pub select: Option<SelectConfig>, - pub watch: Option<WatchConfig>, - pub paths: Option<PathsConfig>, - pub download: Option<DownloadConfig>, - pub update: Option<UpdateConfig>, -} - -#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub(crate) struct GlobalConfig { - pub display_colors: Option<bool>, -} - -#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub(crate) struct UpdateConfig { - pub max_backlog: Option<usize>, -} - -#[derive(Debug, Deserialize, PartialEq, Clone)] -#[serde(deny_unknown_fields)] -pub(crate) struct DownloadConfig { - /// This will then be converted to an u64 - pub max_cache_size: Option<String>, -} - -#[derive(Debug, Deserialize, PartialEq, Clone)] -#[serde(deny_unknown_fields)] -pub(crate) struct SelectConfig { - pub playback_speed: Option<f64>, - pub subtitle_langs: Option<String>, -} - -#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub(crate) struct WatchConfig { - pub local_displays_length: Option<usize>, -} - -#[derive(Debug, Deserialize, PartialEq, Clone)] -#[serde(deny_unknown_fields)] -pub(crate) struct PathsConfig { - pub download_dir: Option<PathBuf>, - pub mpv_config_path: Option<PathBuf>, - pub mpv_input_path: Option<PathBuf>, - pub database_path: Option<PathBuf>, - pub last_selection_path: Option<PathBuf>, -} diff --git a/crates/yt/src/config/file_system.rs b/crates/yt/src/config/file_system.rs deleted file mode 100644 index 2463e9d..0000000 --- a/crates/yt/src/config/file_system.rs +++ /dev/null @@ -1,120 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use crate::config::{DownloadConfig, PathsConfig, SelectConfig, WatchConfig}; - -use super::{ - Config, GlobalConfig, UpdateConfig, - default::{create_path, download, global, paths, select, update, watch}, -}; - -use std::{fs::read_to_string, path::PathBuf}; - -use anyhow::{Context, Result}; -use bytes::Bytes; - -macro_rules! get { - ($default:path, $config:expr, $key_one:ident, $($keys:ident),*) => { - { - let maybe_value = get!{@option $config, $key_one, $($keys),*}; - if let Some(value) = maybe_value { - value - } else { - $default().to_owned() - } - } - }; - - (@option $config:expr, $key_one:ident, $($keys:ident),*) => { - if let Some(key) = $config.$key_one.clone() { - get!{@option key, $($keys),*} - } else { - None - } - }; - (@option $config:expr, $key_one:ident) => { - $config.$key_one - }; - - (@path_if_none $config:expr, $option_default:expr, $default:path, $key_one:ident, $($keys:ident),*) => { - { - let maybe_download_dir: Option<PathBuf> = - get! {@option $config, $key_one, $($keys),*}; - - let down_dir = if let Some(dir) = maybe_download_dir { - PathBuf::from(dir) - } else { - if let Some(path) = $option_default { - path - } else { - $default() - .with_context(|| format!("Failed to get default path for: '{}.{}'", stringify!($key_one), stringify!($($keys),*)))? - } - }; - create_path(down_dir)? - } - }; - (@path $config:expr, $default:path, $key_one:ident, $($keys:ident),*) => { - get! {@path_if_none $config, None, $default, $key_one, $($keys),*} - }; -} - -impl Config { - pub fn from_config_file( - db_path: Option<PathBuf>, - config_path: Option<PathBuf>, - display_colors: Option<bool>, - ) -> Result<Self> { - let config_file_path = - config_path.map_or_else(|| -> Result<_> { paths::config_path() }, Ok)?; - - let config: super::definitions::ConfigFile = - toml::from_str(&read_to_string(config_file_path).unwrap_or(String::new())) - .context("Failed to parse the config file as toml")?; - - Ok(Self { - global: GlobalConfig { - display_colors: { - let config_value: Option<bool> = get! {@option config, global, display_colors}; - - display_colors.unwrap_or(config_value.unwrap_or_else(global::display_colors)) - }, - }, - select: SelectConfig { - playback_speed: get! {select::playback_speed, config, select, playback_speed}, - subtitle_langs: get! {select::subtitle_langs, config, select, subtitle_langs}, - }, - watch: WatchConfig { - local_displays_length: get! {watch::local_displays_length, config, watch, local_displays_length}, - }, - update: UpdateConfig { - max_backlog: get! {update::max_backlog, config, update, max_backlog}, - }, - paths: PathsConfig { - download_dir: get! {@path config, paths::download_dir, paths, download_dir}, - mpv_config_path: get! {@path config, paths::mpv_config_path, paths, mpv_config_path}, - mpv_input_path: get! {@path config, paths::mpv_input_path, paths, mpv_input_path}, - database_path: get! {@path_if_none config, db_path, paths::database_path, paths, database_path}, - last_selection_path: get! {@path config, paths::last_selection_path, paths, last_selection_path}, - }, - download: DownloadConfig { - max_cache_size: { - let bytes_str: String = - get! {download::max_cache_size, config, download, max_cache_size}; - let number: Bytes = bytes_str - .parse() - .context("Failed to parse max_cache_size")?; - number - }, - }, - }) - } -} diff --git a/crates/yt/src/config/mod.rs b/crates/yt/src/config/mod.rs index a10f7c2..05bb4cf 100644 --- a/crates/yt/src/config/mod.rs +++ b/crates/yt/src/config/mod.rs @@ -1,6 +1,5 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> // Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // @@ -9,68 +8,131 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -#![allow(clippy::module_name_repetitions)] +use std::sync::atomic::{AtomicBool, Ordering}; -use std::path::PathBuf; +use crate::config::support::mk_config; -use bytes::Bytes; -use serde::Serialize; +mod non_empty_vec; +mod paths; +mod support; -mod default; -mod definitions; -pub mod file_system; +pub(crate) static SHOULD_DISPLAY_COLOR: AtomicBool = AtomicBool::new(false); -#[derive(Serialize, Debug)] -pub struct Config { - pub global: GlobalConfig, - pub select: SelectConfig, - pub watch: WatchConfig, - pub paths: PathsConfig, - pub download: DownloadConfig, - pub update: UpdateConfig, -} -// These structures could get non-copy fields in the future. +// We need to do both things to comply with what the config expects. +#[allow(clippy::trivially_copy_pass_by_ref, clippy::unnecessary_wraps)] +fn set_static_should_display_color(value: &bool) -> anyhow::Result<()> { + SHOULD_DISPLAY_COLOR.store(*value, Ordering::Relaxed); -#[derive(Serialize, Debug)] -#[allow(missing_copy_implementations)] -pub struct GlobalConfig { - pub display_colors: bool, -} -#[derive(Serialize, Debug)] -#[allow(missing_copy_implementations)] -pub struct UpdateConfig { - pub max_backlog: usize, -} -#[derive(Serialize, Debug)] -#[allow(missing_copy_implementations)] -pub struct DownloadConfig { - pub max_cache_size: Bytes, -} -#[derive(Serialize, Debug)] -pub struct SelectConfig { - pub playback_speed: f64, - pub subtitle_langs: String, -} -#[derive(Serialize, Debug)] -#[allow(missing_copy_implementations)] -pub struct WatchConfig { - pub local_displays_length: usize, -} -#[derive(Serialize, Debug)] -pub struct PathsConfig { - pub download_dir: PathBuf, - pub mpv_config_path: PathBuf, - pub mpv_input_path: PathBuf, - pub database_path: PathBuf, - pub last_selection_path: PathBuf, + Ok(()) } -// pub fn status_path() -> anyhow::Result<PathBuf> { -// const STATUS_PATH: &str = "running.info.json"; -// get_runtime_path(STATUS_PATH) -// } +mk_config! { + use std::path::PathBuf; + use std::io::IsTerminal; + use std::time::Duration; + + use crate::shared::bytes::Bytes; + + use super::set_static_should_display_color; + + use super::paths::get_config_path; + use super::paths::get_runtime_path; + use super::paths::get_data_path; + use super::paths::ensure_parent_dir; + use super::paths::ensure_dir; + use super::paths::PREFIX; + + use super::non_empty_vec::NonEmptyVec; + use super::non_empty_vec::non_empty_vec; + + struct Config { + global: GlobalConfig = { + /// Whether to display colors. + display_colors: bool where display_color: Option<bool> =! {|config_value: Option<bool>| + Ok::<_, anyhow::Error>( + display_color + .unwrap_or( + config_value + .unwrap_or_else(|| std::io::stderr().is_terminal()) + ) + ) + } => set_static_should_display_color, + }, + select: SelectConfig = { + /// The playback speed to use, when it is not overridden. + playback_speed: f64 =: 2.7, + + /// The subtitle langs to download, when it is not overridden. + subtitle_langs: String =: String::new(), + }, + watch: WatchConfig = { + /// How many chars to display at most, when displaying information on mpv's local on screen + /// display. + local_displays_length: usize =: 1000, -// pub fn subscriptions() -> anyhow::Result<PathBuf> { -// const SUBSCRIPTIONS: &str = "subscriptions.json"; -// get_data_path(SUBSCRIPTIONS) -// } + /// How long to wait between saving the video watch progress. + progress_save_intervall: Duration =: Duration::from_secs(10), + }, + commands: CommandsConfig = { + /// Which command to execute, when showing the thumbnail. + /// + /// This command will be executed with the one argument, being the path to the image file to display. + image_show: NonEmptyVec<String> =: non_empty_vec!["imv".to_owned()], + + /// Which command to use, when spawing one of the external commands (e.g. + /// `yt-comments-external` from mpv). + /// + /// The command will be called with a series of args that should be executed. + /// For example, + /// `<your_specified_command> <path_to_yt_binary> --db-path <path_to_current_db_path> comments` + external_spawn: NonEmptyVec<String> =: non_empty_vec!["alacritty".to_owned(), "-e".to_owned()], + + /// Which command to use, when opening video urls (like in the `yt select url` case). + /// + /// This command will be called with one argument, being the url of the video to open. + url_opener: NonEmptyVec<String> =: non_empty_vec!["firefox".to_owned()], + }, + paths: PathsConfig = { + /// Where to store downloaded files. + download_dir: PathBuf =: { + // We download to the temp dir to avoid taxing the disk + let temp_dir = std::env::temp_dir(); + + temp_dir.join(PREFIX) + } => ensure_dir, + + /// Path to the mpv configuration file. + mpv_config_path: PathBuf =? get_config_path("mpv.conf") => ensure_parent_dir, + + /// Path to the mpv input configuration file. + mpv_input_path: PathBuf =? get_config_path("mpv.input.conf") => ensure_parent_dir, + + /// Which path to use for mpv ipc socket creation. + mpv_ipc_socket_path: PathBuf =? get_runtime_path("mpv.ipc.socket") => ensure_parent_dir, + + /// Path to the video database. + database_path: PathBuf where db_path: Option<PathBuf> =! {|config_value: Option<PathBuf>| { + db_path.map_or_else(|| config_value.map_or_else(|| get_data_path("videos.sqlite"), Ok), Ok) + }} => ensure_parent_dir, + + /// Where to store the selection file before applying it. + last_selection_path: PathBuf =? get_runtime_path("selected.yts") => ensure_parent_dir, + }, + download: DownloadConfig = { + /// The maximum cache size. + max_cache_size: Bytes =? "3 GiB".parse(), + }, + update: UpdateConfig = { + /// How many videos to download, when checking for new ones. + max_backlog: usize =: 20, + + /// How many threads to use in the thread pool for fetching new videos. + pool_size: usize =: 16, + + /// How many subscriptions to fetch at once. + /// + /// For example, 16 means, that we will fetch 16 subscriptions at the same time. + futures: usize =: 16 * 4, + }, + } +} diff --git a/crates/yt/src/config/non_empty_vec.rs b/crates/yt/src/config/non_empty_vec.rs new file mode 100644 index 0000000..bd2c5e3 --- /dev/null +++ b/crates/yt/src/config/non_empty_vec.rs @@ -0,0 +1,83 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::VecDeque, + fmt::{Display, Write}, +}; + +use anyhow::bail; +use serde::{Deserialize, Serialize}; + +macro_rules! non_empty_vec { + ($first:expr $(, $($others:expr),+ $(,)?)?) => {{ + let inner: Vec<_> = vec![$first $(, $($others,)+)?]; + inner.try_into().expect("Has a first arg") + }} +} +pub(crate) use non_empty_vec; + +/// A vector that is non-empty. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(try_from = "Vec<T>")] +#[serde(into = "Vec<T>")] +pub(crate) struct NonEmptyVec<T: Clone> { + first: T, + rest: Vec<T>, +} + +impl<T: Clone> TryFrom<Vec<T>> for NonEmptyVec<T> { + type Error = anyhow::Error; + + fn try_from(value: Vec<T>) -> Result<Self, Self::Error> { + let mut queue = VecDeque::from(value); + + if let Some(first) = queue.pop_front() { + Ok(Self { + first, + rest: queue.into(), + }) + } else { + bail!("You need to have at least one element in a non-empty vector.") + } + } +} + +impl<T: Clone> From<NonEmptyVec<T>> for Vec<T> { + fn from(value: NonEmptyVec<T>) -> Self { + let mut base = vec![value.first]; + base.extend(value.rest); + base + } +} + +impl<T: Clone> NonEmptyVec<T> { + pub(crate) fn first(&self) -> &T { + &self.first + } + + pub(crate) fn tail(&self) -> &[T] { + self.rest.as_ref() + } + + pub(crate) fn join(&self, sep: &str) -> String + where + T: Display, + { + let mut output = String::new(); + write!(output, "{}", self.first()).expect("In-memory, does not fail"); + + for elem in &self.rest { + write!(output, "{sep}{elem}").expect("In-memory, does not fail"); + } + + output + } +} diff --git a/crates/yt/src/config/paths.rs b/crates/yt/src/config/paths.rs new file mode 100644 index 0000000..66975dd --- /dev/null +++ b/crates/yt/src/config/paths.rs @@ -0,0 +1,58 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +pub(super) fn get_runtime_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_runtime_file(name) + .with_context(|| format!("Failed to place runtime file: '{name}'")) +} +pub(super) fn get_data_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_data_file(name) + .with_context(|| format!("Failed to place data file: '{name}'")) +} +pub(super) fn get_config_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_config_file(name) + .with_context(|| format!("Failed to place config file: '{name}'")) +} + +pub(super) fn ensure_parent_dir(path: &Path) -> Result<()> { + if !path.exists() { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + } + + Ok(()) +} +pub(super) fn ensure_dir(path: &Path) -> Result<()> { + if !path.exists() { + std::fs::create_dir_all(path) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + + Ok(()) +} + +pub(super) fn config_path() -> Result<PathBuf> { + get_config_path("config.toml") +} + +pub(crate) const PREFIX: &str = "yt"; diff --git a/crates/yt/src/config/support.rs b/crates/yt/src/config/support.rs new file mode 100644 index 0000000..96e7ba4 --- /dev/null +++ b/crates/yt/src/config/support.rs @@ -0,0 +1,161 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +macro_rules! mk_config { + ( + $(use $usage_path:path;)* + + struct $name:ident { + $( + $(#[$attr0:meta])* + $subconfig_name:ident : $subconfig_type:ident = { + $( + $(#[$attr1:meta])* + $field_name:ident : $field_type:ty $( + where $extra_input:ident: $extra_input_type:ty + ),* = $errors:tt $default:expr $(=> $finalizer:ident)? + ),* + $(,)? + } + ),* + $(,)? + } + ) => { + mod _inner { + #![allow(non_snake_case)] + + $(use $usage_path;)* + + #[derive(serde::Serialize, Debug)] + pub(crate) struct $name { + $( + $(#[$attr0])* + pub(crate) $subconfig_name: $subconfig_type + ),* + } + + #[derive(Debug, serde::Deserialize, PartialEq)] + #[serde(deny_unknown_fields)] + #[allow(non_camel_case_types)] + struct config { + $( + $subconfig_name: Option<$subconfig_name> + ),* + } + + impl $name { + pub(crate) fn from_config_file( + config_file_path: Option<std::path::PathBuf>, + $( + $( + $( + $extra_input: $extra_input_type, + )* + )* + )* + ) -> anyhow::Result<Self> { + use anyhow::Context; + + let config_file_path = + config_file_path.map_or_else(|| -> anyhow::Result<_> { super::paths::config_path() }, Ok)?; + + let config: config = + toml::from_str(&std::fs::read_to_string(config_file_path).unwrap_or(String::new())) + .context("Failed to parse the config file as toml")?; + + Ok(Self { + $( + $subconfig_name: { + let toplevel = config.$subconfig_name.unwrap_or_default(); + $subconfig_type { + $( + $field_name: $field_name(toplevel.$field_name, $($extra_input),*)? + ),* + } + } + ),* + }) + } + + pub(crate) fn run_finalizers(&self) -> anyhow::Result<()> { + #[allow(unused_imports)] + use anyhow::Context; + + $( + $( + $( + $finalizer(&self.$subconfig_name.$field_name) + .context( + concat!( + "While running the finalizer for config value '", + stringify!($subconfig_name), + ".", + stringify!($field_name), + "'" + ) + )?; + )? + )* + )* + + Ok(()) + } + } + + $( + #[derive(serde::Serialize, Debug)] + pub(crate) struct $subconfig_type { + $( + $(#[$attr1])* + pub(crate) $field_name: $field_type + ),* + } + + #[derive(Debug, Default, serde::Deserialize, PartialEq)] + #[serde(deny_unknown_fields)] + #[allow(non_camel_case_types)] + struct $subconfig_name { + $( + $field_name: Option<$field_type> + ),* + } + + $( + fn $field_name( + config_value: Option<$field_type>, + $($extra_input: $extra_input_type),* + ) -> anyhow::Result<$field_type> { + use anyhow::Context; + + let expr = $crate::config::support::maybe_wrap_type!($field_type =$errors $default)(config_value); + + expr.context(concat!("Failed to generate default config value for '", stringify!($field_name),"'")) + } + )* + )* + } + pub(crate) use self::_inner::*; + }; +} + +macro_rules! maybe_wrap_type { + ($ty:ty =! $val:expr) => { + (|config_value: Option<$ty>| $val(config_value)) + }; + ($ty:ty =? $val:expr) => { + (|config_value: Option<$ty>| config_value.map_or_else(|| $val, Ok)) + }; + ($ty:ty =: $val:expr) => { + (|config_value: Option<$ty>| Ok::<_, anyhow::Error>(config_value.unwrap_or_else(|| $val))) + }; +} + +pub(crate) use maybe_wrap_type; +pub(crate) use mk_config; diff --git a/crates/yt/src/download/mod.rs b/crates/yt/src/download/mod.rs deleted file mode 100644 index 110bf55..0000000 --- a/crates/yt/src/download/mod.rs +++ /dev/null @@ -1,366 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{collections::HashMap, io, str::FromStr, sync::Arc, time::Duration}; - -use crate::{ - app::App, - download::download_options::download_opts, - storage::video_database::{ - Video, YtDlpOptions, - downloader::{get_next_uncached_video, set_video_cache_path}, - extractor_hash::ExtractorHash, - get::get_video_yt_dlp_opts, - notify::wait_for_cache_reduction, - }, - unreachable::Unreachable, -}; - -use anyhow::{Context, Result, bail}; -use bytes::Bytes; -use futures::{FutureExt, future::BoxFuture}; -use log::{debug, error, info, warn}; -use tokio::{fs, task::JoinHandle, time}; -use yt_dlp::{json_cast, json_get}; - -#[allow(clippy::module_name_repetitions)] -pub mod download_options; -pub mod progress_hook; - -#[derive(Debug)] -#[allow(clippy::module_name_repetitions)] -pub struct CurrentDownload { - task_handle: JoinHandle<Result<()>>, - extractor_hash: ExtractorHash, -} - -impl CurrentDownload { - fn new_from_video(app: Arc<App>, video: Video) -> Self { - let extractor_hash = video.extractor_hash; - - let task_handle = tokio::spawn(async move { - Downloader::actually_cache_video(&app, &video) - .await - .with_context(|| format!("Failed to cache video: '{}'", video.title))?; - Ok(()) - }); - - Self { - task_handle, - extractor_hash, - } - } -} - -enum CacheSizeCheck { - /// The video can be downloaded - Fits, - - /// The video and the current cache size together would exceed the size - TooLarge, - - /// The video would not even fit into the empty cache - ExceedsMaxCacheSize, -} - -#[derive(Debug)] -pub struct Downloader { - current_download: Option<CurrentDownload>, - video_size_cache: HashMap<ExtractorHash, u64>, - printed_warning: bool, - cached_cache_allocation: Option<Bytes>, -} - -impl Default for Downloader { - fn default() -> Self { - Self::new() - } -} - -impl Downloader { - #[must_use] - pub fn new() -> Self { - Self { - current_download: None, - video_size_cache: HashMap::new(), - printed_warning: false, - cached_cache_allocation: None, - } - } - - /// Check if enough cache is available. Will wait for 10s if it's not. - async fn is_enough_cache_available( - &mut self, - app: &App, - max_cache_size: u64, - next_video: &Video, - ) -> Result<CacheSizeCheck> { - if let Some(cdownload) = &self.current_download { - if cdownload.extractor_hash == next_video.extractor_hash { - // If the video is already being downloaded it will always fit. Otherwise the - // download would not have been started. - return Ok(CacheSizeCheck::Fits); - } - } - let cache_allocation = Self::get_current_cache_allocation(app).await?; - let video_size = self.get_approx_video_size(app, next_video)?; - - if video_size >= max_cache_size { - error!( - "The video '{}' ({}) exceeds the maximum cache size ({})! \ - Please set a bigger maximum (`--max-cache-size`) or skip it.", - next_video.title, - Bytes::new(video_size), - Bytes::new(max_cache_size) - ); - - return Ok(CacheSizeCheck::ExceedsMaxCacheSize); - } - - if cache_allocation.as_u64() + video_size >= max_cache_size { - if !self.printed_warning { - warn!( - "Can't download video: '{}' ({}) as it's too large for the cache ({} of {} allocated). \ - Waiting for cache size reduction..", - next_video.title, - Bytes::new(video_size), - &cache_allocation, - Bytes::new(max_cache_size) - ); - self.printed_warning = true; - - // Update this value immediately. - // This avoids printing the "Current cache size has changed .." warning below. - self.cached_cache_allocation = Some(cache_allocation); - } - - if let Some(cca) = self.cached_cache_allocation { - if cca != cache_allocation { - // Only print the warning if the display string has actually changed. - // Otherwise, we might confuse the user - if cca.to_string() != cache_allocation.to_string() { - warn!( - "Current cache size has changed, it's now: '{}'", - cache_allocation - ); - } - debug!( - "Cache size has changed: {} -> {}", - cca.as_u64(), - cache_allocation.as_u64() - ); - self.cached_cache_allocation = Some(cache_allocation); - } - } else { - unreachable!( - "The `printed_warning` should be false in this case, \ - and thus should have already set the `cached_cache_allocation`." - ); - } - - // Wait and hope, that a large video is deleted from the cache. - wait_for_cache_reduction(app).await?; - Ok(CacheSizeCheck::TooLarge) - } else { - self.printed_warning = false; - Ok(CacheSizeCheck::Fits) - } - } - - /// The entry point to the Downloader. - /// This Downloader will periodically check if the database has changed, and then also - /// change which videos it downloads. - /// This will run, until the database doesn't contain any watchable videos - pub async fn consume(&mut self, app: Arc<App>, max_cache_size: u64) -> Result<()> { - while let Some(next_video) = get_next_uncached_video(&app).await? { - match self - .is_enough_cache_available(&app, max_cache_size, &next_video) - .await? - { - CacheSizeCheck::Fits => (), - CacheSizeCheck::TooLarge => continue, - CacheSizeCheck::ExceedsMaxCacheSize => bail!("Giving up."), - }; - - if self.current_download.is_some() { - let current_download = self.current_download.take().unreachable("It is `Some`."); - - if current_download.task_handle.is_finished() { - current_download.task_handle.await??; - continue; - } - - if next_video.extractor_hash == current_download.extractor_hash { - // Reset the taken value - self.current_download = Some(current_download); - } else { - info!( - "Noticed, that the next video is not the video being downloaded, replacing it ('{}' vs. '{}')!", - next_video.extractor_hash.into_short_hash(&app).await?, - current_download - .extractor_hash - .into_short_hash(&app) - .await? - ); - - // Replace the currently downloading video - // FIXME(@bpeetz): This does not work (probably because of the python part.) <2025-02-21> - current_download.task_handle.abort(); - - let new_current_download = - CurrentDownload::new_from_video(Arc::clone(&app), next_video); - - self.current_download = Some(new_current_download); - } - } else { - info!( - "No video is being downloaded right now, setting it to '{}'", - next_video.title - ); - let new_current_download = - CurrentDownload::new_from_video(Arc::clone(&app), next_video); - self.current_download = Some(new_current_download); - } - - // TODO(@bpeetz): Why do we sleep here? <2025-02-21> - time::sleep(Duration::from_secs(1)).await; - } - - info!("Finished downloading!"); - Ok(()) - } - - pub async fn get_current_cache_allocation(app: &App) -> Result<Bytes> { - fn dir_size(mut dir: fs::ReadDir) -> BoxFuture<'static, Result<Bytes>> { - async move { - let mut acc = 0; - while let Some(entry) = dir.next_entry().await? { - let size = match entry.metadata().await? { - data if data.is_dir() => { - let path = entry.path(); - let read_dir = fs::read_dir(path).await?; - - dir_size(read_dir).await?.as_u64() - } - data => data.len(), - }; - acc += size; - } - Ok(Bytes::new(acc)) - } - .boxed() - } - - let read_dir_result = match fs::read_dir(&app.config.paths.download_dir).await { - Ok(ok) => ok, - Err(err) => match err.kind() { - io::ErrorKind::NotFound => { - fs::create_dir_all(&app.config.paths.download_dir) - .await - .with_context(|| { - format!( - "Failed to create download dir at: '{}'", - &app.config.paths.download_dir.display() - ) - })?; - - info!( - "Created empty download dir at '{}'", - &app.config.paths.download_dir.display(), - ); - - // The new dir should not contain anything (otherwise we would not have had to - // create it) - return Ok(Bytes::new(0)); - } - err => Err(io::Error::from(err)).with_context(|| { - format!( - "Failed to get dir size of download dir at: '{}'", - &app.config.paths.download_dir.display() - ) - })?, - }, - }; - - dir_size(read_dir_result).await - } - - fn get_approx_video_size(&mut self, app: &App, video: &Video) -> Result<u64> { - if let Some(value) = self.video_size_cache.get(&video.extractor_hash) { - Ok(*value) - } else { - // the subtitle file size should be negligible - let add_opts = YtDlpOptions { - subtitle_langs: String::new(), - }; - let yt_dlp = download_opts(app, &add_opts)?; - - let result = yt_dlp - .extract_info(&video.url, false, true) - .with_context(|| { - format!("Failed to extract video information: '{}'", video.title) - })?; - - let size = if let Some(val) = result.get("filesize") { - json_cast!(val, as_u64) - } else if let Some(val) = result.get("filesize_approx") { - json_cast!(val, as_u64) - } else if result.get("duration").is_some() && result.get("tbr").is_some() { - #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] - let duration = json_get!(result, "duration", as_f64).ceil() as u64; - - // TODO: yt_dlp gets this from the format - #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] - let tbr = json_get!(result, "tbr", as_f64).ceil() as u64; - - duration * tbr * (1000 / 8) - } else { - let hardcoded_default = Bytes::from_str("250 MiB").expect("This is hardcoded"); - error!( - "Failed to find a filesize for video: '{}' (Using hardcoded value of {})", - video.title, hardcoded_default - ); - hardcoded_default.as_u64() - }; - - assert_eq!( - self.video_size_cache.insert(video.extractor_hash, size), - None - ); - - Ok(size) - } - } - - async fn actually_cache_video(app: &App, video: &Video) -> Result<()> { - debug!("Download started: {}", &video.title); - - let addional_opts = get_video_yt_dlp_opts(app, &video.extractor_hash).await?; - let yt_dlp = download_opts(app, &addional_opts)?; - - let result = yt_dlp - .download(&[video.url.to_owned()]) - .with_context(|| format!("Failed to download video: '{}'", video.title))?; - - assert_eq!(result.len(), 1); - let result = &result[0]; - - set_video_cache_path(app, &video.extractor_hash, Some(result)).await?; - - info!( - "Video '{}' was downlaoded to path: {}", - video.title, - result.display() - ); - - Ok(()) - } -} diff --git a/crates/yt/src/download/progress_hook.rs b/crates/yt/src/download/progress_hook.rs deleted file mode 100644 index b75ec00..0000000 --- a/crates/yt/src/download/progress_hook.rs +++ /dev/null @@ -1,188 +0,0 @@ -use std::{ - io::{Write, stderr}, - process, -}; - -use bytes::Bytes; -use log::{Level, log_enabled}; -use yt_dlp::mk_python_function; - -use crate::{ - ansi_escape_codes::{clear_whole_line, move_to_col}, - select::selection_file::duration::MaybeDuration, -}; - -/// # Panics -/// If expectations fail. -#[allow(clippy::too_many_lines, clippy::needless_pass_by_value)] -pub fn progress_hook( - input: serde_json::Map<String, serde_json::Value>, -) -> Result<(), std::io::Error> { - // Only add the handler, if the log-level is higher than Debug (this avoids covering debug - // messages). - if log_enabled!(Level::Debug) { - return Ok(()); - } - - macro_rules! get { - (@interrogate $item:ident, $type_fun:ident, $get_fun:ident, $name:expr) => {{ - let a = $item.get($name).expect(concat!( - "The field '", - stringify!($name), - "' should exist." - )); - - if a.$type_fun() { - a.$get_fun().expect( - "The should have been checked in the if guard, so unpacking here is fine", - ) - } else { - panic!( - "Value {} => \n{}\n is not of type: {}", - $name, - a, - stringify!($type_fun) - ); - } - }}; - - ($type_fun:ident, $get_fun:ident, $name1:expr, $name2:expr) => {{ - let a = get! {@interrogate input, is_object, as_object, $name1}; - let b = get! {@interrogate a, $type_fun, $get_fun, $name2}; - b - }}; - - ($type_fun:ident, $get_fun:ident, $name:expr) => {{ - get! {@interrogate input, $type_fun, $get_fun, $name} - }}; - } - - macro_rules! default_get { - (@interrogate $item:ident, $default:expr, $get_fun:ident, $name:expr) => {{ - let a = if let Some(field) = $item.get($name) { - field.$get_fun().unwrap_or($default) - } else { - $default - }; - a - }}; - - ($get_fun:ident, $default:expr, $name1:expr, $name2:expr) => {{ - let a = get! {@interrogate input, is_object, as_object, $name1}; - let b = default_get! {@interrogate a, $default, $get_fun, $name2}; - b - }}; - - ($get_fun:ident, $default:expr, $name:expr) => {{ - default_get! {@interrogate input, $default, $get_fun, $name} - }}; - } - - macro_rules! c { - ($color:expr, $format:expr) => { - format!("\x1b[{}m{}\x1b[0m", $color, $format) - }; - } - - #[allow(clippy::items_after_statements)] - fn format_bytes(bytes: u64) -> String { - let bytes = Bytes::new(bytes); - bytes.to_string() - } - - #[allow(clippy::items_after_statements)] - fn format_speed(speed: f64) -> String { - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - let bytes = Bytes::new(speed.floor() as u64); - format!("{bytes}/s") - } - - let get_title = || -> String { - match get! {is_string, as_str, "info_dict", "ext"} { - "vtt" => { - format!( - "Subtitles ({})", - default_get! {as_str, "<No Subtitle Language>", "info_dict", "name"} - ) - } - "webm" | "mp4" | "mp3" | "m4a" => { - default_get! { as_str, "<No title>", "info_dict", "title"}.to_owned() - } - other => panic!("The extension '{other}' is not yet implemented"), - } - }; - - match get! {is_string, as_str, "status"} { - "downloading" => { - let elapsed = default_get! {as_f64, 0.0f64, "elapsed"}; - let eta = default_get! {as_f64, 0.0, "eta"}; - let speed = default_get! {as_f64, 0.0, "speed"}; - - let downloaded_bytes = get! {is_u64, as_u64, "downloaded_bytes"}; - let (total_bytes, bytes_is_estimate): (u64, &'static str) = { - let total_bytes = default_get!(as_u64, 0, "total_bytes"); - if total_bytes == 0 { - let maybe_estimate = default_get!(as_u64, 0, "total_bytes_estimate"); - - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - if maybe_estimate == 0 { - // The download speed should be in bytes per second and the eta in seconds. - // Thus multiplying them gets us the raw bytes (which were estimated by `yt_dlp`, from their `info.json`) - let bytes_still_needed = (speed * eta).ceil() as u64; - - (downloaded_bytes + bytes_still_needed, "~") - } else { - (maybe_estimate, "~") - } - } else { - (total_bytes, "") - } - }; - - let percent: f64 = { - if total_bytes == 0 { - 100.0 - } else { - #[allow( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::cast_precision_loss - )] - { - (downloaded_bytes as f64 / total_bytes as f64) * 100.0 - } - } - }; - - clear_whole_line(); - move_to_col(1); - - eprint!( - "'{}' [{}/{} at {}] -> [{} of {}{} {}] ", - c!("34;1", get_title()), - c!("33;1", MaybeDuration::from_secs_f64(elapsed)), - c!("33;1", MaybeDuration::from_secs_f64(eta)), - c!("32;1", format_speed(speed)), - c!("31;1", format_bytes(downloaded_bytes)), - c!("31;1", bytes_is_estimate), - c!("31;1", format_bytes(total_bytes)), - c!("36;1", format!("{:.02}%", percent)) - ); - stderr().flush()?; - } - "finished" => { - eprintln!("-> Finished downloading."); - } - "error" => { - // TODO: This should probably return an Err. But I'm not so sure where the error would - // bubble up to (i.e., who would catch it) <2025-01-21> - eprintln!("-> Error while downloading: {}", get_title()); - process::exit(1); - } - other => unreachable!("'{other}' should not be a valid state!"), - } - - Ok(()) -} - -mk_python_function!(progress_hook, wrapped_progress_hook); diff --git a/crates/yt/src/main.rs b/crates/yt/src/main.rs index 930d269..705e642 100644 --- a/crates/yt/src/main.rs +++ b/crates/yt/src/main.rs @@ -11,51 +11,35 @@ // `yt` is not a library. Besides, the `anyhow::Result` type is really useless, if you're not going // to print it anyways. -#![allow(clippy::missing_errors_doc)] +#![allow(clippy::missing_errors_doc, clippy::missing_panics_doc)] -use std::{env::current_exe, sync::Arc}; - -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result}; use app::App; -use bytes::Bytes; -use cache::{invalidate, maintain}; -use clap::Parser; -use cli::{CacheCommand, SelectCommand, SubscriptionCommand, VideosCommand}; +use clap::{CommandFactory, Parser}; use config::Config; -use log::{error, info}; -use select::cmds::handle_select_cmd; -use storage::video_database::get::video_by_hash; -use tokio::{ - fs::File, - io::{BufReader, stdin}, - task::JoinHandle, -}; +use log::info; + +use crate::commands::Command; -use crate::{cli::Command, storage::subscriptions}; +pub(crate) mod output; +pub(crate) mod yt_dlp; -pub mod ansi_escape_codes; -pub mod app; -pub mod cli; -pub mod unreachable; +pub(crate) mod ansi_escape_codes; +pub(crate) mod app; +pub(crate) mod cli; +pub(crate) mod commands; +pub(crate) mod shared; -pub mod cache; -pub mod comments; -pub mod config; -pub mod constants; -pub mod download; -pub mod select; -pub mod status; -pub mod storage; -pub mod subscribe; -pub mod update; -pub mod version; -pub mod videos; -pub mod watch; +pub(crate) mod config; +pub(crate) mod select; +pub(crate) mod storage; +pub(crate) mod version; +pub(crate) mod videos; #[tokio::main] -// This is _the_ main function after all. It is not really good, but it sort of works. -#[allow(clippy::too_many_lines)] async fn main() -> Result<()> { + clap_complete::CompleteEnv::with_factory(cli::CliArgs::command).complete(); + let args = cli::CliArgs::parse(); // The default verbosity is 1 (Warn) @@ -82,219 +66,24 @@ async fn main() -> Result<()> { } }); - let config = Config::from_config_file(args.db_path, args.config_path, args.color)?; + let config = Config::from_config_file(args.config_path, args.color, args.db_path)?; if args.version { version::show(&config).await?; return Ok(()); } - let app = App::new(config, !args.no_migrate_db).await?; - - match args.command.unwrap_or(Command::default()) { - Command::Download { - force, - max_cache_size, - } => { - let max_cache_size = - max_cache_size.unwrap_or(app.config.download.max_cache_size.as_u64()); - info!("Max cache size: '{}'", Bytes::new(max_cache_size)); - - maintain(&app, false).await?; - if force { - invalidate(&app, true).await?; - } - - download::Downloader::new() - .consume(Arc::new(app), max_cache_size) - .await?; - } - Command::Select { cmd } => { - let cmd = cmd.unwrap_or(SelectCommand::default()); - - match cmd { - SelectCommand::File { - done, - use_last_selection, - split, - } => { - if split { - assert!(!use_last_selection); - Box::pin(select::select_split(&app, done)).await? - } else { - Box::pin(select::select_file(&app, done, use_last_selection)).await? - } - } - _ => Box::pin(handle_select_cmd(&app, cmd, None)).await?, - } - } - Command::Sedowa {} => { - Box::pin(select::select_file(&app, false, false)).await?; - - let arc_app = Arc::new(app); - dowa(arc_app).await?; - } - Command::Dowa {} => { - let arc_app = Arc::new(app); - dowa(arc_app).await?; - } - Command::Videos { cmd } => match cmd { - VideosCommand::List { - search_query, - limit, - } => { - videos::query(&app, limit, search_query) - .await - .context("Failed to query videos")?; - } - VideosCommand::Info { hash } => { - let video = video_by_hash(&app, &hash.realize(&app).await?).await?; - - print!( - "{}", - &video - .to_info_display(&app) - .await - .context("Failed to format video")? - ); - } - }, - Command::Update { - max_backlog, - subscriptions, - grouped, - current_progress, - total_number, - } => { - let all_subs = subscriptions::get(&app).await?; - - for sub in &subscriptions { - if !all_subs.0.contains_key(sub) { - bail!( - "Your specified subscription to update '{}' is not a subscription!", - sub - ) - } - } - - let max_backlog = max_backlog.unwrap_or(app.config.update.max_backlog); - - if grouped { - const CHUNK_SIZE: usize = 50; - - assert!(current_progress.is_none() && total_number.is_none()); - - let subs = { - if subscriptions.is_empty() { - all_subs.0.into_iter().map(|sub| sub.0).collect() - } else { - subscriptions - } - }; - - let total_number = subs.len(); - let mut current_progress = 0; - for chunk in subs.chunks(CHUNK_SIZE) { - info!( - "$ yt update {}", - chunk - .iter() - .map(|sub_name| format!("{sub_name:#?}")) - .collect::<Vec<_>>() - .join(" ") - ); - - let status = std::process::Command::new( - current_exe().context("Failed to get the current exe to re-execute")?, - ) - .arg("update") - .args(["--current-progress", current_progress.to_string().as_str()]) - .args(["--total-number", total_number.to_string().as_str()]) - .args(chunk) - .status()?; - - if !status.success() { - bail!("grouped yt update: Child process failed."); - } - - current_progress += CHUNK_SIZE; - } - } else { - update::update(&app, max_backlog, subscriptions, total_number, current_progress).await?; - } - } - Command::Subscriptions { cmd } => match cmd { - SubscriptionCommand::Add { name, url } => { - subscribe::subscribe(&app, name, url) - .await - .context("Failed to add a subscription")?; - } - SubscriptionCommand::Remove { name } => { - subscribe::unsubscribe(&app, name) - .await - .context("Failed to remove a subscription")?; - } - SubscriptionCommand::List {} => { - let all_subs = subscriptions::get(&app).await?; - - for (key, val) in all_subs.0 { - println!("{}: '{}'", key, val.url); - } - } - SubscriptionCommand::Export {} => { - let all_subs = subscriptions::get(&app).await?; - for val in all_subs.0.values() { - println!("{}", val.url); - } - } - SubscriptionCommand::Import { file, force } => { - if let Some(file) = file { - let f = File::open(file).await?; - - subscribe::import(&app, BufReader::new(f), force).await?; - } else { - subscribe::import(&app, BufReader::new(stdin()), force).await?; - } - } - }, - - Command::Watch {} => watch::watch(Arc::new(app)).await?, - Command::Playlist { watch } => watch::playlist::playlist(&app, watch).await?, - - Command::Status {} => status::show(&app).await?, - Command::Config {} => status::config(&app)?, - - Command::Database { command } => match command { - CacheCommand::Invalidate { hard } => invalidate(&app, hard).await?, - CacheCommand::Maintain { all } => maintain(&app, all).await?, - }, - - Command::Comments {} => { - comments::comments(&app).await?; - } - Command::Description {} => { - comments::description(&app).await?; - } - } - - Ok(()) -} - -async fn dowa(arc_app: Arc<App>) -> Result<()> { - let max_cache_size = arc_app.config.download.max_cache_size; - info!("Max cache size: '{max_cache_size}'"); + // Perform config finalization _after_ checking for the version + // so that version always works. + config + .run_finalizers() + .context("Failed to finalize config for usage")?; - let arc_app_clone = Arc::clone(&arc_app); - let download: JoinHandle<()> = tokio::spawn(async move { - let result = download::Downloader::new() - .consume(arc_app_clone, max_cache_size.as_u64()) - .await; + let app = App::new(config, !args.no_migrate_db).await?; - if let Err(err) = result { - error!("Error from downloader: {err:?}"); - } - }); + args.command + .unwrap_or(Command::default()) + .implm(app) + .await?; - watch::watch(arc_app).await?; - download.await?; Ok(()) } diff --git a/crates/yt/src/comments/output.rs b/crates/yt/src/output/mod.rs index cb3a9c4..2f74519 100644 --- a/crates/yt/src/comments/output.rs +++ b/crates/yt/src/output/mod.rs @@ -17,9 +17,7 @@ use std::{ use anyhow::{Context, Result}; use uu_fmt::{FmtOptions, process_text}; -use crate::unreachable::Unreachable; - -pub async fn display_fmt_and_less(input: String) -> Result<()> { +pub(crate) fn display_less(input: String) -> Result<()> { let mut less = Command::new("less") .args(["--raw-control-chars"]) .stdin(Stdio::piped()) @@ -27,12 +25,11 @@ pub async fn display_fmt_and_less(input: String) -> Result<()> { .spawn() .context("Failed to run less")?; - let input = format_text(&input); let mut stdin = less.stdin.take().context("Failed to open stdin")?; std::thread::spawn(move || { stdin .write_all(input.as_bytes()) - .unreachable("Should be able to write to the stdin of less"); + .expect("Should be able to write to the stdin of less"); }); let _ = less.wait().context("Failed to await less")?; @@ -40,9 +37,15 @@ pub async fn display_fmt_and_less(input: String) -> Result<()> { Ok(()) } +pub(crate) fn display_fmt_and_less(input: &str) -> Result<()> { + display_less(format_text(&input, None)) +} + #[must_use] -pub fn format_text(input: &str) -> String { - let width = termsize::get().map_or(90, |size| size.cols); +pub(crate) fn format_text(input: &str, termsize: Option<u16>) -> String { + let input = input.trim(); + + let width = termsize.unwrap_or_else(|| termsize::get().map_or(90, |size| size.cols)); let fmt_opts = FmtOptions { uniform: true, split_only: true, diff --git a/crates/yt/src/select/cmds/mod.rs b/crates/yt/src/select/cmds/mod.rs deleted file mode 100644 index aabcd3d..0000000 --- a/crates/yt/src/select/cmds/mod.rs +++ /dev/null @@ -1,111 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use crate::{ - app::App, - cli::{SelectCommand, SharedSelectionCommandArgs}, - storage::video_database::{ - Priority, VideoOptions, VideoStatus, - get::video_by_hash, - set::{set_video_options, video_status}, - }, -}; - -use anyhow::{Context, Result, bail}; - -mod add; - -pub async fn handle_select_cmd( - app: &App, - cmd: SelectCommand, - line_number: Option<i64>, -) -> Result<()> { - match cmd { - SelectCommand::Pick { shared } => { - handle_status_change(app, shared, line_number, VideoStatus::Pick).await?; - } - SelectCommand::Drop { shared } => { - handle_status_change(app, shared, line_number, VideoStatus::Drop).await?; - } - SelectCommand::Watched { shared } => { - handle_status_change(app, shared, line_number, VideoStatus::Watched).await?; - } - SelectCommand::Add { urls, start, stop } => { - Box::pin(add::add(app, urls, start, stop)).await?; - } - SelectCommand::Watch { shared } => { - let hash = shared.hash.clone().realize(app).await?; - - let video = video_by_hash(app, &hash).await?; - - if let VideoStatus::Cached { - cache_path, - is_focused, - } = video.status - { - handle_status_change( - app, - shared, - line_number, - VideoStatus::Cached { - cache_path, - is_focused, - }, - ) - .await?; - } else { - handle_status_change(app, shared, line_number, VideoStatus::Watch).await?; - } - } - - SelectCommand::Url { shared } => { - let Some(url) = shared.url else { - bail!("You need to provide a url to `select url ..`") - }; - - let mut firefox = std::process::Command::new("firefox"); - firefox.args(["-P", "timesinks.youtube"]); - firefox.arg(url.as_str()); - let _handle = firefox.spawn().context("Failed to run firefox")?; - } - SelectCommand::File { .. } => unreachable!("This should have been filtered out"), - } - Ok(()) -} - -async fn handle_status_change( - app: &App, - shared: SharedSelectionCommandArgs, - line_number: Option<i64>, - new_status: VideoStatus, -) -> Result<()> { - let hash = shared.hash.realize(app).await?; - let video_options = VideoOptions::new( - shared - .subtitle_langs - .unwrap_or(app.config.select.subtitle_langs.clone()), - shared.speed.unwrap_or(app.config.select.playback_speed), - ); - let priority = compute_priority(line_number, shared.priority); - - video_status(app, &hash, new_status, priority).await?; - set_video_options(app, &hash, &video_options).await?; - - Ok(()) -} - -fn compute_priority(line_number: Option<i64>, priority: Option<i64>) -> Option<Priority> { - if let Some(pri) = priority { - Some(Priority::from(pri)) - } else { - line_number.map(Priority::from) - } -} diff --git a/crates/yt/src/select/selection_file/duration.rs b/crates/yt/src/select/duration.rs index 668a0b8..f1de2ea 100644 --- a/crates/yt/src/select/selection_file/duration.rs +++ b/crates/yt/src/select/duration.rs @@ -20,51 +20,45 @@ const HOUR: u64 = 60 * MINUTE; const DAY: u64 = 24 * HOUR; #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct MaybeDuration { +pub(crate) struct MaybeDuration { time: Option<Duration>, } impl MaybeDuration { #[must_use] - pub fn from_std(d: Duration) -> Self { + pub(crate) fn from_std(d: Duration) -> Self { Self { time: Some(d) } } #[must_use] - pub fn from_secs_f64(d: f64) -> Self { + pub(crate) fn from_secs_f64(d: f64) -> Self { Self { time: Some(Duration::from_secs_f64(d)), } } #[must_use] - pub fn from_maybe_secs_f64(d: Option<f64>) -> Self { + pub(crate) fn from_maybe_secs_f64(d: Option<f64>) -> Self { Self { time: d.map(Duration::from_secs_f64), } } #[must_use] - pub fn from_secs(d: u64) -> Self { + #[cfg(test)] + pub(crate) fn from_secs(d: u64) -> Self { Self { time: Some(Duration::from_secs(d)), } } - #[must_use] - pub fn zero() -> Self { - Self { - time: Some(Duration::default()), - } - } - /// Try to return the current duration encoded as seconds. #[must_use] - pub fn as_secs(&self) -> Option<u64> { + pub(crate) fn as_secs(&self) -> Option<u64> { self.time.map(|v| v.as_secs()) } /// Try to return the current duration encoded as seconds and nanoseconds. #[must_use] - pub fn as_secs_f64(&self) -> Option<f64> { + pub(crate) fn as_secs_f64(&self) -> Option<f64> { self.time.map(|v| v.as_secs_f64()) } } @@ -209,7 +203,7 @@ impl std::fmt::Display for MaybeDuration { mod test { use std::str::FromStr; - use crate::select::selection_file::duration::{DAY, HOUR, MINUTE}; + use crate::select::duration::{DAY, HOUR, MINUTE}; use super::MaybeDuration; diff --git a/crates/yt/src/select/mod.rs b/crates/yt/src/select/mod.rs index 668ab02..b02677f 100644 --- a/crates/yt/src/select/mod.rs +++ b/crates/yt/src/select/mod.rs @@ -9,259 +9,7 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use std::{ - collections::HashMap, - env::{self}, - fs::{self, File}, - io::{BufRead, BufReader, BufWriter, Write}, - iter, - path::Path, - string::String, -}; - -use crate::{ - app::App, - cli::CliArgs, - constants::HELP_STR, - storage::video_database::{Video, VideoStatusMarker, get}, - unreachable::Unreachable, -}; - -use anyhow::{Context, Result, bail}; -use clap::Parser; -use cmds::handle_select_cmd; -use futures::{TryStreamExt, stream::FuturesOrdered}; -use log::info; -use selection_file::process_line; -use tempfile::Builder; -use tokio::process::Command; - -pub mod cmds; -pub mod selection_file; - -pub async fn select_split(app: &App, done: bool) -> Result<()> { - let temp_dir = Builder::new() - .prefix("yt_video_select-") - .rand_bytes(6) - .tempdir() - .context("Failed to get tempdir")?; - - let matching_videos = get_videos(app, done).await?; - - let mut no_author = vec![]; - let mut author_map = HashMap::new(); - for video in matching_videos { - if let Some(sub) = &video.parent_subscription_name { - if author_map.contains_key(sub) { - let vec: &mut Vec<_> = author_map - .get_mut(sub) - .unreachable("This key is set, we checked in the if above"); - - vec.push(video); - } else { - author_map.insert(sub.to_owned(), vec![video]); - } - } else { - no_author.push(video); - } - } - - let author_map = { - let mut temp_vec: Vec<_> = author_map.into_iter().collect(); - - // PERFORMANCE: The clone here should not be neeed. <2025-06-15> - temp_vec.sort_by_key(|(name, _)| name.to_owned()); - - temp_vec - }; - - for (index, (name, videos)) in author_map - .into_iter() - .chain(iter::once(( - "<No parent subscription>".to_owned(), - no_author, - ))) - .enumerate() - { - let mut file_path = temp_dir.path().join(format!("{index:02}_{name}")); - file_path.set_extension("yts"); - - let tmp_file = File::create(&file_path) - .with_context(|| format!("Falied to create file at: {}", file_path.display()))?; - - write_videos_to_file(app, &tmp_file, &videos) - .await - .with_context(|| format!("Falied to populate file at: {}", file_path.display()))?; - } - - open_editor_at(temp_dir.path()).await?; - - let mut paths = vec![]; - for maybe_entry in temp_dir - .path() - .read_dir() - .context("Failed to open temp dir for reading")? - { - let entry = maybe_entry.context("Failed to read entry in temp dir")?; - - if !entry.file_type()?.is_file() { - bail!("Found non-file entry: {}", entry.path().display()); - } - - paths.push(entry.path()); - } - - paths.sort(); - - let mut processed = 0; - for path in paths { - let read_file = File::open(path)?; - processed = process_file(app, &read_file, processed).await?; - } - - info!("Processed {processed} records."); - temp_dir.close().context("Failed to close the temp dir")?; - Ok(()) -} - -pub async fn select_file(app: &App, done: bool, use_last_selection: bool) -> Result<()> { - let temp_file = Builder::new() - .prefix("yt_video_select-") - .suffix(".yts") - .rand_bytes(6) - .tempfile() - .context("Failed to get tempfile")?; - - if use_last_selection { - fs::copy(&app.config.paths.last_selection_path, &temp_file)?; - } else { - let matching_videos = get_videos(app, done).await?; - - write_videos_to_file(app, temp_file.as_file(), &matching_videos).await?; - } - - open_editor_at(temp_file.path()).await?; - - let read_file = temp_file.reopen()?; - fs::copy(temp_file.path(), &app.config.paths.last_selection_path) - .context("Failed to persist selection file")?; - - let processed = process_file(app, &read_file, 0).await?; - info!("Processed {processed} records."); - - Ok(()) -} - -async fn get_videos(app: &App, include_done: bool) -> Result<Vec<Video>> { - if include_done { - get::videos(app, VideoStatusMarker::ALL).await - } else { - get::videos( - app, - &[ - VideoStatusMarker::Pick, - // - VideoStatusMarker::Watch, - VideoStatusMarker::Cached, - ], - ) - .await - } -} - -async fn write_videos_to_file(app: &App, file: &File, videos: &[Video]) -> Result<()> { - // Warm-up the cache for the display rendering of the videos. - // Otherwise the futures would all try to warm it up at the same time. - if let Some(vid) = videos.first() { - drop(vid.to_line_display(app).await?); - } - - let mut edit_file = BufWriter::new(file); - - videos - .iter() - .map(|vid| vid.to_select_file_display(app)) - .collect::<FuturesOrdered<_>>() - .try_collect::<Vec<String>>() - .await? - .into_iter() - .try_for_each(|line| -> Result<()> { - edit_file - .write_all(line.as_bytes()) - .context("Failed to write to `edit_file`")?; - - Ok(()) - })?; - - edit_file.write_all(HELP_STR.as_bytes())?; - edit_file.flush().context("Failed to flush edit file")?; - - Ok(()) -} - -async fn process_file(app: &App, file: &File, processed: i64) -> Result<i64> { - let reader = BufReader::new(file); - - let mut line_number = -processed; - - for line in reader.lines() { - let line = line.context("Failed to read a line")?; - - if let Some(line) = process_line(&line)? { - line_number -= 1; - - // debug!( - // "Parsed command: `{}`", - // line.iter() - // .map(|val| format!("\"{}\"", val)) - // .collect::<Vec<String>>() - // .join(" ") - // ); - - let arg_line = ["yt", "select"] - .into_iter() - .chain(line.iter().map(String::as_str)); - - let args = CliArgs::parse_from(arg_line); - - let crate::cli::Command::Select { cmd } = args - .command - .unreachable("This will be some, as we constructed it above.") - else { - unreachable!("This is checked in the `filter_line` function") - }; - - Box::pin(handle_select_cmd( - app, - cmd.unreachable( - "This value should always be some \ - here, as it would otherwise thrown an error above.", - ), - Some(line_number), - )) - .await?; - } - } - - Ok(line_number * -1) -} - -async fn open_editor_at(path: &Path) -> Result<()> { - let editor = env::var("EDITOR").unwrap_or("nvim".to_owned()); - - let mut nvim = Command::new(&editor); - nvim.arg(path); - let status = nvim - .status() - .await - .with_context(|| format!("Falied to run editor: {editor}"))?; - - if status.success() { - Ok(()) - } else { - bail!("Editor ({editor}) exited with error status: {}", status) - } -} +pub(crate) mod duration; // // FIXME: There should be no reason why we need to re-run yt, just to get the help string. But I've // // yet to find a way to do it without the extra exec <2024-08-20> diff --git a/crates/yt/src/select/selection_file/mod.rs b/crates/yt/src/select/selection_file/mod.rs deleted file mode 100644 index abd26c4..0000000 --- a/crates/yt/src/select/selection_file/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -//! The data structures needed to express the file, which the user edits - -use anyhow::{Context, Result}; -use trinitry::Trinitry; - -pub mod duration; - -pub fn process_line(line: &str) -> Result<Option<Vec<String>>> { - // Filter out comments and empty lines - if line.starts_with('#') || line.trim().is_empty() { - Ok(None) - } else { - let tri = Trinitry::new(line).with_context(|| format!("Failed to parse line '{line}'"))?; - - let mut vec = Vec::with_capacity(tri.arguments().len() + 1); - vec.push(tri.command().to_owned()); - vec.extend(tri.arguments().to_vec()); - - Ok(Some(vec)) - } -} diff --git a/crates/bytes/src/error.rs b/crates/yt/src/shared/bytes/error.rs index c9783d8..c9783d8 100644 --- a/crates/bytes/src/error.rs +++ b/crates/yt/src/shared/bytes/error.rs diff --git a/crates/bytes/src/lib.rs b/crates/yt/src/shared/bytes/mod.rs index 2a9248d..31e782e 100644 --- a/crates/bytes/src/lib.rs +++ b/crates/yt/src/shared/bytes/mod.rs @@ -16,6 +16,7 @@ )] use std::{fmt::Display, str::FromStr}; +use ::serde::{Deserialize, Serialize}; use error::BytesError; const B: u64 = 1; @@ -31,10 +32,11 @@ const MB: u64 = 1000 * KB; const GB: u64 = 1000 * MB; const TB: u64 = 1000 * GB; -pub mod error; -pub mod serde; +pub(crate) mod error; -#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq, Deserialize, Serialize)] +#[serde(try_from = "String")] +#[serde(into = "String")] pub struct Bytes(u64); impl Bytes { @@ -131,6 +133,20 @@ impl Display for Bytes { } } +impl From<Bytes> for String { + fn from(value: Bytes) -> Self { + value.to_string() + } +} + +impl TryFrom<String> for Bytes { + type Error = BytesError; + + fn try_from(value: String) -> Result<Self, Self::Error> { + value.as_str().parse() + } +} + // taken from this stack overflow question: https://stackoverflow.com/a/76572321 /// Round to significant digits (rather than digits after the decimal). /// @@ -149,7 +165,7 @@ impl Display for Bytes { ///# } /// ``` #[must_use] -pub fn precision_f64(x: f64, decimals: u32) -> f64 { +pub(crate) fn precision_f64(x: f64, decimals: u32) -> f64 { if x == 0. || decimals == 0 { 0. } else { diff --git a/crates/yt/src/shared/mod.rs b/crates/yt/src/shared/mod.rs new file mode 100644 index 0000000..d3cc563 --- /dev/null +++ b/crates/yt/src/shared/mod.rs @@ -0,0 +1,11 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod bytes; diff --git a/crates/yt/src/status/mod.rs b/crates/yt/src/status/mod.rs deleted file mode 100644 index 18bef7d..0000000 --- a/crates/yt/src/status/mod.rs +++ /dev/null @@ -1,129 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::time::Duration; - -use crate::{ - app::App, - download::Downloader, - select::selection_file::duration::MaybeDuration, - storage::{ - subscriptions, - video_database::{VideoStatusMarker, get}, - }, -}; - -use anyhow::{Context, Result}; -use bytes::Bytes; - -macro_rules! get { - ($videos:expr, $status:ident) => { - $videos - .iter() - .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) - .count() - }; - - (@collect $videos:expr, $status:ident) => { - $videos - .iter() - .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) - .collect() - }; -} - -pub async fn show(app: &App) -> Result<()> { - let all_videos = get::videos(app, VideoStatusMarker::ALL).await?; - - // lengths - let picked_videos_len = get!(all_videos, Pick); - - let watch_videos_len = get!(all_videos, Watch); - let cached_videos_len = get!(all_videos, Cached); - let watched_videos_len = get!(all_videos, Watched); - let watched_videos: Vec<_> = get!(@collect all_videos, Watched); - - let drop_videos_len = get!(all_videos, Drop); - let dropped_videos_len = get!(all_videos, Dropped); - - let subscriptions = subscriptions::get(app).await?; - let subscriptions_len = subscriptions.0.len(); - - let watchtime_status = { - let total_watch_time_raw = watched_videos - .iter() - .fold(Duration::default(), |acc, vid| acc + vid.watch_progress); - - // Most things are watched at a speed of s (which is defined in the config file). - // Thus - // y = x * s -> y / s = x - let total_watch_time = Duration::from_secs_f64( - (total_watch_time_raw.as_secs_f64()) / app.config.select.playback_speed, - ); - - let speed = app.config.select.playback_speed; - - // Do not print the adjusted time, if the user has keep the speed level at 1. - #[allow(clippy::float_cmp)] - if speed == 1.0 { - format!( - "Total Watchtime: {}\n", - MaybeDuration::from_std(total_watch_time_raw) - ) - } else { - format!( - "Total Watchtime: {} (at {speed} speed: {})\n", - MaybeDuration::from_std(total_watch_time_raw), - MaybeDuration::from_std(total_watch_time), - ) - } - }; - - let watch_rate: f64 = { - fn to_f64(input: usize) -> f64 { - f64::from(u32::try_from(input).expect("This should never exceed u32::MAX")) - } - - let count = to_f64(watched_videos_len) / (to_f64(drop_videos_len) + to_f64(dropped_videos_len)); - count * 100.0 - }; - - let cache_usage_raw = Downloader::get_current_cache_allocation(app) - .await - .context("Failed to get current cache allocation")?; - let cache_usage: Bytes = cache_usage_raw; - println!( - "\ -Picked Videos: {picked_videos_len} - -Watch Videos: {watch_videos_len} -Cached Videos: {cached_videos_len} -Watched Videos: {watched_videos_len} (watch rate: {watch_rate:.2} %) - -Drop Videos: {drop_videos_len} -Dropped Videos: {dropped_videos_len} - -{watchtime_status} - - Subscriptions: {subscriptions_len} - Cache usage: {cache_usage}" - ); - - Ok(()) -} - -pub fn config(app: &App) -> Result<()> { - let config_str = toml::to_string(&app.config)?; - - print!("{config_str}"); - - Ok(()) -} diff --git a/crates/yt/src/storage/db/extractor_hash.rs b/crates/yt/src/storage/db/extractor_hash.rs new file mode 100644 index 0000000..3ad8273 --- /dev/null +++ b/crates/yt/src/storage/db/extractor_hash.rs @@ -0,0 +1,220 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashSet, fmt::Display, str::FromStr}; + +use anyhow::{Context, Result, bail}; +use blake3::Hash; +use log::debug; +use serde::{Deserialize, Serialize}; +use tokio::sync::OnceCell; +use yt_dlp::{info_json::InfoJson, json_cast, json_get, json_try_get}; + +use crate::app::App; + +static EXTRACTOR_HASH_LENGTH: OnceCell<usize> = OnceCell::const_new(); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, Serialize, Deserialize)] +pub(crate) struct ExtractorHash { + hash: Hash, +} + +impl Display for ExtractorHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.hash.fmt(f) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ShortHash(String); + +impl Display for ShortHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone)] +#[allow(clippy::module_name_repetitions)] +pub(crate) struct LazyExtractorHash { + value: ShortHash, +} + +impl FromStr for LazyExtractorHash { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { + // perform some cheap validation + if s.len() > 64 { + bail!("A hash can only contain 64 bytes!"); + } + + Ok(Self { + value: ShortHash(s.to_owned()), + }) + } +} + +impl LazyExtractorHash { + /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] + pub(crate) async fn realize( + self, + app: &App, + all_hashes: Option<&[ExtractorHash]>, + ) -> Result<ExtractorHash> { + ExtractorHash::from_short_hash(app, &self.value, all_hashes).await + } +} + +impl ExtractorHash { + #[must_use] + pub(crate) fn from_hash(hash: Hash) -> Self { + Self { hash } + } + + pub(crate) async fn from_short_hash( + app: &App, + s: &ShortHash, + all_hashes: Option<&[Self]>, + ) -> Result<Self> { + let all_hashes = if let Some(all) = all_hashes { + all + } else { + &Self::get_all(app) + .await + .context("Failed to fetch all extractor-hashes from the database")? + }; + let needed_chars = s.0.len(); + for hash in all_hashes { + // PERFORMANCE(@bpeetz): This could avoid the string construction and just use a + // numeric equality check instead. <2025-07-15> + if hash.hash().to_hex()[..needed_chars] == s.0 { + return Ok(*hash); + } + } + bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); + } + + pub(crate) fn from_info_json(entry: &InfoJson) -> Self { + // HACK(@bpeetz): The code that follows is a gross hack. + // One would expect the `id` to be unique _and_ constant for each and every possible info JSON. + // But .. it's just not. The `ARDMediathek` extractor, will sometimes return different `id`s for the same + // video, effectively causing us to insert the same video again into the db (which fails, + // because the URL is still unique). + // + // As such we _should_ probably find a constant value for all extractors, but that just does + // not exist currently, without processing each entry (which is expensive and which I would + // like to avoid). + // + // Therefor, we simply special case the `ARDBetaMediathek` extractor. <2025-07-04> + + // NOTE(@bpeetz): `yt-dlp` apparently uses these two different names for the same thing <2025-07-04> + let ie_key = { + if let Some(ie_key) = json_try_get!(entry, "ie_key", as_str) { + ie_key + } else if let Some(extractor_key) = json_try_get!(entry, "extractor_key", as_str) { + extractor_key + } else { + unreachable!( + "Either `ie_key` or `extractor_key` \ + should be set on every entry info json" + ) + } + }; + + if ie_key == "ARDBetaMediathek" { + // NOTE(@bpeetz): The mediathek is changing their Id scheme, from an `short` old Id to the + // new id. As the new id is too long for some people, yt-dlp will be default return the old + // one (when it is still available!). The new one is called `display_id`. + // Therefore, we simply check if the new one is explicitly returned, and otherwise use the + // normal `id` value, as these are cases where the old one is no longer available. <2025-07-04> + let id = if let Some(display_id) = json_try_get!(entry, "display_id", as_str) { + display_id.as_bytes() + } else { + json_get!(entry, "id", as_str).as_bytes() + }; + + Self { + hash: blake3::hash(id), + } + } else { + Self { + hash: blake3::hash(json_get!(entry, "id", as_str).as_bytes()), + } + } + } + + #[must_use] + pub(crate) fn hash(&self) -> &Hash { + &self.hash + } + + pub(crate) async fn as_short_hash(&self, app: &App) -> Result<ShortHash> { + let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { + *needed_chars + } else { + let needed_chars = self + .get_needed_char_len(app) + .await + .context("Failed to calculate needed char length")?; + EXTRACTOR_HASH_LENGTH + .set(needed_chars) + .expect("This should work at this stage, as we checked above that it is empty."); + + needed_chars + }; + + Ok(ShortHash( + self.hash() + .to_hex() + .chars() + .take(needed_chars) + .collect::<String>(), + )) + } + + async fn get_needed_char_len(&self, app: &App) -> Result<usize> { + debug!("Calculating the needed hash char length"); + let all_hashes = Self::get_all(app) + .await + .context("Failed to fetch all extractor -hashesh from database")?; + + let all_char_vec_hashes = all_hashes + .into_iter() + .map(|hash| hash.hash().to_hex().chars().collect::<Vec<char>>()) + .collect::<Vec<Vec<_>>>(); + + // This value should be updated later, if not rust will panic in the assertion. + let mut needed_chars: usize = 1000; + 'outer: for i in 1..64 { + let i_chars: Vec<String> = all_char_vec_hashes + .iter() + .map(|vec| vec.iter().take(i).collect::<String>()) + .collect(); + + let mut uniqnes_hashmap: HashSet<String> = HashSet::new(); + for ch in i_chars { + if !uniqnes_hashmap.insert(ch) { + // The key was already in the hash map, thus we have a duplicated char and need + // at least one char more + continue 'outer; + } + } + + needed_chars = i; + break 'outer; + } + + assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); + + Ok(needed_chars) + } +} diff --git a/crates/yt/src/storage/db/get/extractor_hash.rs b/crates/yt/src/storage/db/get/extractor_hash.rs new file mode 100644 index 0000000..c8e150a --- /dev/null +++ b/crates/yt/src/storage/db/get/extractor_hash.rs @@ -0,0 +1,68 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use blake3::Hash; +use sqlx::{SqliteConnection, query}; + +use crate::{ + app::App, + storage::db::{ + extractor_hash::ExtractorHash, + video::{Video, video_from_record}, + }, +}; + +impl ExtractorHash { + pub(crate) async fn get(&self, txn: &mut SqliteConnection) -> Result<Video> { + let extractor_hash = self.hash().to_string(); + + let base = query!( + r#" + SELECT * + FROM videos + WHERE extractor_hash = ? + "#, + extractor_hash + ) + .fetch_one(txn) + .await?; + + Ok(video_from_record!(base)) + } + + pub(crate) async fn get_with_app(&self, app: &App) -> Result<Video> { + let mut txn = app.database.begin().await?; + let out = self.get(&mut txn).await?; + txn.commit().await?; + + Ok(out) + } + + pub(crate) async fn get_all(app: &App) -> Result<Vec<Self>> { + let hashes_hex = query!( + r#" + SELECT extractor_hash + FROM videos; + "# + ) + .fetch_all(&app.database) + .await?; + + Ok(hashes_hex + .iter() + .map(|hash| { + Self::from_hash(Hash::from_hex(&hash.extractor_hash).expect( + "These values started as blake3 hashes, they should stay blake3 hashes", + )) + }) + .collect()) + } +} diff --git a/crates/yt/src/storage/db/get/mod.rs b/crates/yt/src/storage/db/get/mod.rs new file mode 100644 index 0000000..4bcd066 --- /dev/null +++ b/crates/yt/src/storage/db/get/mod.rs @@ -0,0 +1,15 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod extractor_hash; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod txn_log; +pub(crate) mod video; diff --git a/crates/yt/src/storage/db/get/playlist.rs b/crates/yt/src/storage/db/get/playlist.rs new file mode 100644 index 0000000..5094523 --- /dev/null +++ b/crates/yt/src/storage/db/get/playlist.rs @@ -0,0 +1,68 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{ + playlist::{Playlist, PlaylistIndex}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +impl Playlist { + /// Get an video based in its index. + #[must_use] + pub(crate) fn get_mut(&mut self, index: PlaylistIndex) -> Option<&mut Video> { + self.videos.get_mut(Into::<usize>::into(index)) + } + + /// Create a playlist, by loading it from the database. + pub(crate) async fn create(app: &App) -> Result<Self> { + let videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + Ok(Self { videos }) + } + + /// Return the current playlist index. + /// + /// This effectively looks for the currently focused video and returns it's index. + /// + /// # Panics + /// Only if internal assertions fail. + pub(crate) fn current_index(&self) -> Option<PlaylistIndex> { + if let Some((index, _)) = self.get_focused() { + Some(index) + } else { + None + } + } + + /// Get the currently focused video, if it exists. + #[must_use] + pub(crate) fn get_focused_mut(&mut self) -> Option<(PlaylistIndex, &mut Video)> { + self.videos + .iter_mut() + .enumerate() + .find(|(_, v)| v.is_focused()) + .map(|(index, video)| (PlaylistIndex::from(index), video)) + } + + /// Get the currently focused video, if it exists. + #[must_use] + pub(crate) fn get_focused(&self) -> Option<(PlaylistIndex, &Video)> { + self.videos + .iter() + .enumerate() + .find(|(_, v)| v.is_focused()) + .map(|(index, video)| (PlaylistIndex::from(index), video)) + } +} diff --git a/crates/yt/src/storage/db/get/subscription.rs b/crates/yt/src/storage/db/get/subscription.rs new file mode 100644 index 0000000..1d0b660 --- /dev/null +++ b/crates/yt/src/storage/db/get/subscription.rs @@ -0,0 +1,65 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::collections::HashMap; + +use crate::{ + app::App, + storage::db::subscription::{Subscription, Subscriptions}, +}; + +use anyhow::Result; +use sqlx::query; +use url::Url; + +impl Subscriptions { + /// Get a list of subscriptions + pub(crate) async fn get(app: &App) -> Result<Self> { + let raw_subs = query!( + " + SELECT * + FROM subscriptions; + " + ) + .fetch_all(&app.database) + .await?; + + let subscriptions: HashMap<String, Subscription> = raw_subs + .into_iter() + .map(|sub| { + ( + sub.name.clone(), + Subscription::new( + sub.name, + Url::parse(&sub.url).expect("It was an URL, when we inserted it."), + if sub.is_active == 1 { + true + } else if sub.is_active == 0 { + false + } else { + unreachable!("These are the only two options") + }, + ), + ) + }) + .collect(); + + Ok(Subscriptions(subscriptions)) + } + + pub(crate) fn remove_inactive(self) -> Self { + Self( + self.0 + .into_iter() + .filter(|(_, sub)| sub.is_active) + .collect(), + ) + } +} diff --git a/crates/yt/src/storage/db/get/txn_log.rs b/crates/yt/src/storage/db/get/txn_log.rs new file mode 100644 index 0000000..1a6df2c --- /dev/null +++ b/crates/yt/src/storage/db/get/txn_log.rs @@ -0,0 +1,43 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{insert::Committable, txn_log::TxnLog, video::TimeStamp}, +}; + +use anyhow::Result; +use sqlx::query; + +impl<O: Committable> TxnLog<O> { + /// Get the log of all operations that have been performed. + pub(crate) async fn get(app: &App) -> Result<Self> { + let raw_ops = query!( + " + SELECT * + FROM txn_log + ORDER BY timestamp ASC; + " + ) + .fetch_all(&app.database) + .await?; + + let inner = raw_ops + .into_iter() + .filter_map(|raw_op| { + serde_json::from_str(&raw_op.operation) + .map(|parsed_op| (TimeStamp::from_secs(raw_op.timestamp), parsed_op)) + .ok() + }) + .collect(); + + Ok(TxnLog::new(inner)) + } +} diff --git a/crates/yt/src/storage/db/get/video/mod.rs b/crates/yt/src/storage/db/get/video/mod.rs new file mode 100644 index 0000000..69adb6b --- /dev/null +++ b/crates/yt/src/storage/db/get/video/mod.rs @@ -0,0 +1,261 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{fs::File, path::PathBuf}; + +use anyhow::{Context, Result, bail}; +use log::debug; +use sqlx::query; +use yt_dlp::{info_json::InfoJson, json_cast, json_try_get}; + +use crate::{ + app::App, + storage::db::video::{ + Video, VideoStatus, VideoStatusMarker, + comments::{Comments, raw::RawComment}, + video_from_record, + }, +}; + +impl Video { + /// Returns to next video which should be downloaded. This respects the priority assigned by select. + /// It does not return videos, which are already downloaded. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) async fn next_to_download(app: &App) -> Result<Option<Self>> { + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + // NOTE: The ORDER BY statement should be the same as the one in [`in_states`]. <2024-08-22> + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND cache_path IS NULL + ORDER BY priority DESC, publish_date DESC + LIMIT 1; + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record!(base))) + } + } + + /// Returns the description of the current video. + /// The returned description will be set to `<No description>` in the absence of one. + /// + /// # Errors + /// If no current video exists. + /// + /// # Panics + /// If the current video lacks the `info.json` file. + pub(crate) async fn get_current_description(app: &App) -> Result<String> { + let Some(currently_playing_video) = Video::currently_focused(app).await? else { + bail!("Could not find a currently playing video!"); + }; + + let info_json = ¤tly_playing_video.get_info_json()?.expect( + "A currently *playing* must be cached. \ + And thus the info.json should be available.", + ); + + let description = json_try_get!(info_json, "description", as_str) + .unwrap_or("<No description>") + .to_owned(); + + Ok(description) + } + + /// Returns the comments of the current video. + /// The returned [`Comments`] will be empty in the absence of comments. + /// + /// # Errors + /// If no current video exists. + /// + /// # Panics + /// If the current video lacks the `info.json` file. + pub(crate) async fn get_current_comments(app: &App) -> Result<Comments> { + let Some(currently_playing_video) = Video::currently_focused(app).await? else { + bail!("Could not find a currently playing video!"); + }; + + let info_json = ¤tly_playing_video.get_info_json()?.expect( + "A currently *playing* video must be cached. \ + And thus the info.json should be available.", + ); + + let raw_comments = if let Some(comments) = json_try_get!(info_json, "comments", as_array) { + comments + .iter() + .cloned() + .map(serde_json::from_value) + .collect::<Result<Vec<RawComment>, _>>()? + } else { + // TODO(@bpeetz): We could display a `<No-comments>` here. <2025-07-15> + + bail!( + "The video ('{}') does not have comments!", + json_try_get!(info_json, "title", as_str).unwrap_or("<No Title>") + ) + }; + + Ok(Comments::from_raw(raw_comments)) + } + + /// Optionally returns the video that is currently focused. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) async fn currently_focused(app: &App) -> Result<Option<Self>> { + let status = VideoStatusMarker::Cached.as_db_integer(); + + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND is_focused = 1 + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record!(base))) + } + } + + /// Calculate the [`info_json`] location on-disk for this video. + /// + /// Will return [`None`], if the video does not have an downloaded [`info_json`] + pub(crate) fn info_json_path(&self) -> Result<Option<PathBuf>> { + if let VideoStatus::Cached { mut cache_path, .. } = self.status.clone() { + if !cache_path.set_extension("info.json") { + bail!( + "Failed to change path extension to 'info.json': {}", + cache_path.display() + ); + } + + Ok(Some(cache_path)) + } else { + Ok(None) + } + } + + /// Fetch the [`info_json`], downloaded on-disk for this video. + /// + /// Will return [`None`], if the video does not have an downloaded [`info_json`] + pub(crate) fn get_info_json(&self) -> Result<Option<InfoJson>> { + if let Some(path) = self.info_json_path()? { + let info_json_string = File::open(path)?; + let info_json = serde_json::from_reader(&info_json_string)?; + + Ok(Some(info_json)) + } else { + Ok(None) + } + } + + /// Returns this videos `is_focused` flag if it is set. + /// + /// Will return `false` for not-downloaded videos. + pub(crate) fn is_focused(&self) -> bool { + if let VideoStatus::Cached { is_focused, .. } = &self.status { + *is_focused + } else { + false + } + } + + /// Returns the videos that are in the `allowed_states`. + /// + /// # Panics + /// Only, if assertions fail. + pub(crate) async fn in_states( + app: &App, + allowed_states: &[VideoStatusMarker], + ) -> Result<Vec<Video>> { + fn test(all_states: &[VideoStatusMarker], check: VideoStatusMarker) -> Option<i64> { + if all_states.contains(&check) { + Some(check.as_db_integer()) + } else { + None + } + } + fn states_to_string(allowed_states: &[VideoStatusMarker]) -> String { + let mut states = allowed_states + .iter() + .fold(String::from("&["), |mut acc, state| { + acc.push_str(state.as_str()); + acc.push_str(", "); + acc + }); + states = states.trim().to_owned(); + states = states.trim_end_matches(',').to_owned(); + states.push(']'); + states + } + + debug!( + "Fetching videos in the states: '{}'", + states_to_string(allowed_states) + ); + let active_pick: Option<i64> = test(allowed_states, VideoStatusMarker::Pick); + let active_watch: Option<i64> = test(allowed_states, VideoStatusMarker::Watch); + let active_cached: Option<i64> = test(allowed_states, VideoStatusMarker::Cached); + let active_watched: Option<i64> = test(allowed_states, VideoStatusMarker::Watched); + let active_drop: Option<i64> = test(allowed_states, VideoStatusMarker::Drop); + let active_dropped: Option<i64> = test(allowed_states, VideoStatusMarker::Dropped); + + // NOTE: The ORDER BY statement should be the same as the one in [`next_to_download`]. <2024-08-22> + let videos = query!( + r" + SELECT * + FROM videos + WHERE status IN (?,?,?,?,?,?) + ORDER BY priority DESC, publish_date DESC; + ", + active_pick, + active_watch, + active_cached, + active_watched, + active_drop, + active_dropped, + ) + .fetch_all(&app.database) + .await + .with_context(|| { + format!( + "Failed to query videos with states: '{}'", + states_to_string(allowed_states) + ) + })?; + + let real_videos: Vec<Video> = videos + .iter() + .map(|base| -> Video { video_from_record!(base) }) + .collect(); + + Ok(real_videos) + } +} diff --git a/crates/yt/src/storage/db/insert/maintenance.rs b/crates/yt/src/storage/db/insert/maintenance.rs new file mode 100644 index 0000000..d87c1ae --- /dev/null +++ b/crates/yt/src/storage/db/insert/maintenance.rs @@ -0,0 +1,38 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{ + insert::Operations, + video::{Video, VideoStatus, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +/// Remove the downloaded paths from videos in the db, that no longer exist on the file system. +pub(crate) async fn clear_stale_downloaded_paths(app: &App) -> Result<()> { + let mut cached_videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("DbMaintain: init"); + for vid in &mut cached_videos { + if let VideoStatus::Cached { cache_path, .. } = &vid.status { + if !cache_path.exists() { + vid.remove_download_path(&mut ops); + } + } else { + unreachable!("We only asked for cached videos.") + } + } + ops.commit(app).await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/db/insert/mod.rs b/crates/yt/src/storage/db/insert/mod.rs new file mode 100644 index 0000000..3458608 --- /dev/null +++ b/crates/yt/src/storage/db/insert/mod.rs @@ -0,0 +1,115 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::mem; + +use crate::app::App; + +use anyhow::Result; +use chrono::Utc; +use log::{debug, trace}; +use serde::{Serialize, de::DeserializeOwned}; +use sqlx::{SqliteConnection, query}; + +pub(crate) mod maintenance; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod video; + +pub(crate) trait Committable: + Sized + std::fmt::Debug + Serialize + DeserializeOwned +{ + async fn commit(self, txn: &mut SqliteConnection) -> Result<()>; +} + +#[derive(Debug)] +pub(crate) struct Operations<O: Committable> { + name: &'static str, + ops: Vec<O>, +} + +impl<O: Committable> Default for Operations<O> { + fn default() -> Self { + Self::new("<default impl>") + } +} + +impl<O: Committable> Operations<O> { + #[must_use] + pub(crate) fn new(name: &'static str) -> Self { + Self { + name, + ops: Vec::new(), + } + } + + pub(crate) async fn commit(mut self, app: &App) -> Result<()> { + let ops = mem::take(&mut self.ops); + + if ops.is_empty() { + return Ok(()); + } + + trace!("Begin commit of {}", self.name); + let mut txn = app.database.begin().await?; + + for op in ops { + trace!("Commiting operation: {op:?}"); + add_operation_to_txn_log(&op, &mut txn).await?; + op.commit(&mut txn).await?; + } + + txn.commit().await?; + trace!("End commit of {}", self.name); + + Ok(()) + } + + pub(crate) fn push(&mut self, op: O) { + self.ops.push(op); + } +} + +impl<O: Committable> Drop for Operations<O> { + fn drop(&mut self) { + assert!( + self.ops.is_empty(), + "Trying to drop uncommitted operations (name: {}) ({:#?}). This is a bug.", + self.name, + self.ops + ); + } +} + +async fn add_operation_to_txn_log<O: Committable>( + operation: &O, + txn: &mut SqliteConnection, +) -> Result<()> { + debug!("Adding operation to txn log: {operation:?}"); + + let now = Utc::now().timestamp(); + let operation = serde_json::to_string(&operation).expect("should be serializable"); + + query!( + r#" + INSERT INTO txn_log ( + timestamp, + operation + ) + VALUES (?, ?); + "#, + now, + operation, + ) + .execute(txn) + .await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/db/insert/playlist.rs b/crates/yt/src/storage/db/insert/playlist.rs new file mode 100644 index 0000000..4d3e140 --- /dev/null +++ b/crates/yt/src/storage/db/insert/playlist.rs @@ -0,0 +1,222 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{cmp::Ordering, time::Duration}; + +use anyhow::{Context, Result}; +use colors::Colorize; +use libmpv2::Mpv; +use log::{debug, trace}; + +use crate::{ + app::App, + storage::db::{ + insert::{Operations, video::Operation}, + playlist::{Playlist, PlaylistIndex}, + video::VideoStatus, + }, +}; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum VideoTransition { + Watched, + Picked, +} + +impl Playlist { + pub(crate) fn mark_current_done( + &mut self, + app: &App, + mpv: &Mpv, + new_state: VideoTransition, + ops: &mut Operations<Operation>, + ) -> Result<()> { + let (current_index, current_video) = self + .get_focused_mut() + .expect("This should be some at this point"); + + debug!( + "Playlist handler will mark video '{}' {:?}.", + current_video.title, new_state + ); + + match new_state { + VideoTransition::Watched => current_video.set_watched(ops), + VideoTransition::Picked => current_video.set_status(VideoStatus::Pick, ops), + } + + self.save_watch_progress(mpv, current_index, ops); + + self.videos.remove(Into::<usize>::into(current_index)); + + { + // Decide which video to mark focused now. + let index = usize::from(current_index); + let playlist_length = self.len(); + + if playlist_length == 0 { + // There are no new videos to mark focused. + } else { + let index = match index.cmp(&playlist_length) { + Ordering::Greater => { + unreachable!( + "The index '{index}' cannot exceed the \ + playlist length '{playlist_length}' as indices are 0 based." + ); + } + Ordering::Less => { + // The index is still valid. + // Therefore, we keep the user at this position. + index + } + Ordering::Equal => { + // The index is pointing to the end of the playlist. We could either go the second + // to last entry (i.e., one entry back) or wrap around to the start. + // We wrap around. + 0 + } + }; + + let next = self + .get_mut(PlaylistIndex::from(index)) + .expect("We checked that the index is still good"); + next.set_focused(true, ops); + } + + // Tell mpv about our decision. + self.resync_with_mpv(app, mpv)?; + } + + Ok(()) + } + + /// Sync the mpv playlist with this playlist. + pub(crate) fn resync_with_mpv(&self, app: &App, mpv: &Mpv) -> Result<()> { + fn get_playlist_count(mpv: &Mpv) -> Result<usize> { + mpv.get_property::<i64>("playlist/count") + .context("Failed to get mpv playlist len") + .map(|count| { + usize::try_from(count).expect("The playlist_count should always be positive") + }) + } + + if get_playlist_count(mpv)? != 0 { + // We could also use `loadlist`, but that would require use to start a unix socket or even + // write all the video paths to a file beforehand + mpv.command("playlist-clear", &[])?; + mpv.command("playlist-remove", &["current"])?; + } + + assert_eq!( + get_playlist_count(mpv)?, + 0, + "The playlist should be empty at this point." + ); + + debug!("MpvReload: Adding {} videos to playlist.", self.len()); + + self.videos + .iter() + .enumerate() + .try_for_each(|(index, video)| { + let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + else { + unreachable!("All of the videos in a playlist are cached"); + }; + + let options = format!( + "speed={},start={}", + video + .playback_speed + .unwrap_or(app.config.select.playback_speed), + i64::try_from(video.watch_progress.as_secs()) + .expect("This should not overflow"), + ); + + mpv.command( + "loadfile", + &[ + cache_path.to_str().with_context(|| { + format!( + "Failed to parse the video cache path ('{}') as valid utf8", + cache_path.display() + ) + })?, + "append-play", + "-1", // Not used for `append-play`, but needed for the next args to take effect. + options.as_str(), + ], + )?; + + if *is_focused { + debug!("MpvReload: Setting playlist position to {index}"); + mpv.set_property("playlist-pos", index.to_string().as_str())?; + } + + Ok::<(), anyhow::Error>(()) + })?; + + Ok(()) + } + + pub(crate) fn save_current_watch_progress( + &mut self, + mpv: &Mpv, + ops: &mut Operations<Operation>, + ) { + let (index, _) = self + .get_focused_mut() + .expect("This should be some at this point"); + + self.save_watch_progress(mpv, index, ops); + } + + /// Saves the `watch_progress` of a video at the index. + pub(crate) fn save_watch_progress( + &mut self, + mpv: &Mpv, + at: PlaylistIndex, + ops: &mut Operations<Operation>, + ) { + let current_video = self + .get_mut(at) + .expect("We should never produce invalid playlist indices"); + + let watch_progress = match mpv.get_property::<i64>("time-pos") { + Ok(time) => u64::try_from(time) + .expect("This conversion should never fail as the `time-pos` property is positive"), + Err(err) => { + // We cannot hard error here, as we would open us to an race condition between mpv + // changing the current video and we saving it. + trace!( + "While trying to save the watch progress for the current video: \ + Failed to get the watchprogress of the currently playling video: \ + (This is probably expected, nevertheless showing the raw error) \ + {err}" + ); + + return; + } + }; + + let watch_progress = Duration::from_secs(watch_progress); + + debug!( + "Setting the watch progress for the current_video '{}' to {}s", + current_video.title_fmt().render(false), + watch_progress.as_secs(), + ); + + current_video.set_watch_progress(watch_progress, ops); + } +} diff --git a/crates/yt/src/storage/db/insert/subscription.rs b/crates/yt/src/storage/db/insert/subscription.rs new file mode 100644 index 0000000..54409a9 --- /dev/null +++ b/crates/yt/src/storage/db/insert/subscription.rs @@ -0,0 +1,128 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::storage::db::{ + insert::{Committable, Operations}, + subscription::{Subscription, Subscriptions}, +}; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use sqlx::query; + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum Operation { + Add(Subscription), + Remove(Subscription), + SetIsActive { + target: Subscription, + is_active: bool, + }, +} + +impl Committable for Operation { + async fn commit(self, txn: &mut sqlx::SqliteConnection) -> Result<()> { + match self { + Operation::Add(subscription) => { + let url = subscription.url.as_str(); + + query!( + " + INSERT INTO subscriptions ( + name, + url + ) VALUES (?, ?); + ", + subscription.name, + url + ) + .execute(txn) + .await?; + + println!( + "Subscribed to '{}' at '{}'", + subscription.name, subscription.url + ); + Ok(()) + } + Operation::Remove(subscription) => { + let output = query!( + " + DELETE FROM subscriptions + WHERE name = ? + ", + subscription.name, + ) + .execute(txn) + .await?; + + assert_eq!( + output.rows_affected(), + 1, + "The removed subscription query did effect more (or less) than one row. This is a bug." + ); + + println!( + "Unsubscribed from '{}' at '{}'", + subscription.name, subscription.url + ); + + Ok(()) + } + Operation::SetIsActive { target, is_active } => { + query!( + " + UPDATE subscriptions + SET is_active = ? + WHERE name = ?; + ", + is_active, + target.name + ) + .execute(txn) + .await?; + + println!( + "Marked '{}' as '{}'", + target.name, + if is_active { "active" } else { "inactive" } + ); + Ok(()) + } + } + } +} + +impl Subscription { + pub(crate) fn add(self, ops: &mut Operations<Operation>) { + ops.push(Operation::Add(self)); + } + + pub(crate) fn remove(self, ops: &mut Operations<Operation>) { + ops.push(Operation::Remove(self)); + } + + pub(crate) fn set_is_active(self, is_active: bool, ops: &mut Operations<Operation>) { + if self.is_active != is_active { + ops.push(Operation::SetIsActive { + target: self, + is_active, + }); + } + } +} + +impl Subscriptions { + pub(crate) fn remove(self, ops: &mut Operations<Operation>) { + for sub in self.0.into_values() { + ops.push(Operation::Remove(sub)); + } + } +} diff --git a/crates/yt/src/storage/db/insert/video/mod.rs b/crates/yt/src/storage/db/insert/video/mod.rs new file mode 100644 index 0000000..da62e37 --- /dev/null +++ b/crates/yt/src/storage/db/insert/video/mod.rs @@ -0,0 +1,610 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + path::{Path, PathBuf}, + time, +}; + +use crate::storage::db::{ + extractor_hash::ExtractorHash, + insert::{Committable, Operations}, + video::{Priority, Video, VideoStatus, VideoStatusMarker}, +}; + +use anyhow::{Context, Result}; +use chrono::Utc; +use log::debug; +use serde::{Deserialize, Serialize}; +use sqlx::query; +use tokio::fs; + +use super::super::video::TimeStamp; + +const fn is_focused_to_value(is_focused: bool) -> Option<i8> { + if is_focused { Some(1) } else { None } +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum Operation { + Add { + description: Option<String>, + title: String, + parent_subscription_name: Option<String>, + thumbnail_url: Option<String>, + url: String, + extractor_hash: String, + status: i64, + cache_path: Option<String>, + is_focused: Option<i8>, + duration: Option<f64>, + last_status_change: i64, + publish_date: Option<i64>, + watch_progress: i64, + }, + // TODO(@bpeetz): Could both the {`Set`,`Remove`}`DownloadPath` ops, be merged into SetStatus + // {`Cached`,`Watch`}? <2025-07-14> + SetDownloadPath { + video: ExtractorHash, + path: PathBuf, + }, + RemoveDownloadPath { + video: ExtractorHash, + }, + SetStatus { + video: ExtractorHash, + status: VideoStatus, + }, + SetPriority { + video: ExtractorHash, + priority: Priority, + }, + SetPlaybackSpeed { + video: ExtractorHash, + playback_speed: f64, + }, + SetSubtitleLangs { + video: ExtractorHash, + subtitle_langs: String, + }, + SetWatchProgress { + video: ExtractorHash, + watch_progress: time::Duration, + }, + SetIsFocused { + video: ExtractorHash, + is_focused: bool, + }, +} + +impl Committable for Operation { + #[allow(clippy::too_many_lines)] + async fn commit(self, txn: &mut sqlx::SqliteConnection) -> Result<()> { + match self { + Operation::SetDownloadPath { video, path } => { + debug!("Setting cache path from '{video}' to '{}'", path.display()); + + let path_str = path.display().to_string(); + let extractor_hash = video.hash().to_string(); + let status = VideoStatusMarker::Cached.as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = ?, status = ? + WHERE extractor_hash = ?; + "#, + path_str, + status, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::RemoveDownloadPath { video } => { + let extractor_hash = video.hash().to_string(); + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + let old = video.get(&mut *txn).await?; + + debug!("Deleting download path of '{video}' ({}).", old.title); + + if let VideoStatus::Cached { cache_path, .. } = &old.status { + if let Ok(true) = cache_path.try_exists() { + fs::remove_file(cache_path).await?; + } + + { + let info_json_path = old.info_json_path()?.expect("Is downloaded"); + + if let Ok(true) = info_json_path.try_exists() { + fs::remove_file(info_json_path).await?; + } + } + + { + if old.subtitle_langs.is_some() { + // TODO(@bpeetz): Also clean-up the downloaded subtitle files. <2025-07-05> + } + } + } else { + unreachable!( + "A video cannot have a download path deletion \ + queued without being marked as Cached." + ); + } + + query!( + r#" + UPDATE videos + SET cache_path = NULL, status = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + status, + None::<i32>, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetStatus { video, status } => { + let extractor_hash = video.hash().to_string(); + + let old = video.get(&mut *txn).await?; + + let old_marker = old.status.as_marker(); + + let (cache_path, is_focused) = { + fn cache_path_to_string(path: &Path) -> Result<String> { + Ok(path + .to_str() + .with_context(|| { + format!( + "Failed to parse cache path ('{}') as utf8 string", + path.display() + ) + })? + .to_owned()) + } + + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &status + { + ( + Some(cache_path_to_string(cache_path)?), + is_focused_to_value(*is_focused), + ) + } else { + (None, None) + } + }; + + let new_status = status.as_marker(); + + assert_ne!( + old_marker, new_status, + "We should have never generated this operation" + ); + + let now = Utc::now().timestamp(); + + debug!( + "Changing status of video ('{}' {extractor_hash}) \ + from {old_marker:#?} to {new_status:#?}.", + old.title + ); + + let new_status = new_status.as_db_integer(); + query!( + r#" + UPDATE videos + SET status = ?, last_status_change = ?, cache_path = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + new_status, + now, + cache_path, + is_focused, + extractor_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetPriority { video, priority } => { + let extractor_hash = video.hash().to_string(); + + let new_priority = priority.as_db_integer(); + query!( + r#" + UPDATE videos + SET priority = ? + WHERE extractor_hash = ?; + "#, + new_priority, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetWatchProgress { + video, + watch_progress, + } => { + let video_extractor_hash = video.hash().to_string(); + let watch_progress = i64::try_from(watch_progress.as_secs()) + .expect("This should never exceed its bounds"); + + query!( + r#" + UPDATE videos + SET watch_progress = ? + WHERE extractor_hash = ?; + "#, + watch_progress, + video_extractor_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetPlaybackSpeed { + video, + playback_speed, + } => { + let extractor_hash = video.hash().to_string(); + + query!( + r#" + UPDATE videos + SET playback_speed = ? + WHERE extractor_hash = ?; + "#, + playback_speed, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetSubtitleLangs { + video, + subtitle_langs, + } => { + let extractor_hash = video.hash().to_string(); + + query!( + r#" + UPDATE videos + SET subtitle_langs = ? + WHERE extractor_hash = ?; + "#, + subtitle_langs, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetIsFocused { video, is_focused } => { + debug!("Set is_focused of video: '{video}' to {is_focused}"); + let new_hash = video.hash().to_string(); + let new_is_focused = is_focused_to_value(is_focused); + + query!( + r#" + UPDATE videos + SET is_focused = ? + WHERE extractor_hash = ?; + "#, + new_is_focused, + new_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::Add { + parent_subscription_name, + thumbnail_url, + url, + extractor_hash, + status, + cache_path, + is_focused, + duration, + last_status_change, + publish_date, + watch_progress, + description, + title, + } => { + query!( + r#" + INSERT INTO videos ( + description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + title, + url, + watch_progress, + cache_path + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + "#, + description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + title, + url, + watch_progress, + cache_path, + ) + .execute(txn) + .await?; + + Ok(()) + } + } + } +} + +impl Video { + /// Add this in-memory video to the db. + pub(crate) fn add(self, ops: &mut Operations<Operation>) -> Result<Self> { + let description = self.description.clone(); + let title = self.title.clone(); + let parent_subscription_name = self.parent_subscription_name.clone(); + + let thumbnail_url = self.thumbnail_url.as_ref().map(ToString::to_string); + + let url = self.url.to_string(); + let extractor_hash = self.extractor_hash.hash().to_string(); + + let status = self.status.as_marker().as_db_integer(); + let (cache_path, is_focused) = if let VideoStatus::Cached { + cache_path, + is_focused, + } = &self.status + { + ( + Some( + cache_path + .to_str() + .with_context(|| { + format!( + "Failed to prase cache path '{}' as utf-8 string", + cache_path.display() + ) + })? + .to_string(), + ), + is_focused_to_value(*is_focused), + ) + } else { + (None, None) + }; + + let duration: Option<f64> = self.duration.as_secs_f64(); + let last_status_change: i64 = self.last_status_change.as_secs(); + let publish_date: Option<i64> = self.publish_date.map(TimeStamp::as_secs); + let watch_progress: i64 = + i64::try_from(self.watch_progress.as_secs()).expect("This should never exceed a u32"); + + ops.push(Operation::Add { + description, + title, + parent_subscription_name, + thumbnail_url, + url, + extractor_hash, + status, + cache_path, + is_focused, + duration, + last_status_change, + publish_date, + watch_progress, + }); + + Ok(self) + } + + /// Set the download path of a video. + /// + /// # Note + /// This will also set the status to `Cached`. + pub(crate) fn set_download_path(&mut self, path: &Path, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { cache_path, .. } = &mut self.status { + if cache_path != path { + // Update the in-memory video. + path.clone_into(cache_path); + + ops.push(Operation::SetDownloadPath { + video: self.extractor_hash, + path: path.to_owned(), + }); + } + } else { + self.status = VideoStatus::Cached { + cache_path: path.to_owned(), + is_focused: false, + }; + + ops.push(Operation::SetDownloadPath { + video: self.extractor_hash, + path: path.to_owned(), + }); + } + } + + /// Remove the download path of a video. + /// + /// # Note + /// This will also set the status to `Watch`. + /// + /// # Panics + /// If the status is not `Cached`. + pub(crate) fn remove_download_path(&mut self, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { .. } = &mut self.status { + self.status = VideoStatus::Watch; + ops.push(Operation::RemoveDownloadPath { + video: self.extractor_hash, + }); + } else { + unreachable!("Can only remove the path from a `Cached` video"); + } + } + + /// Update the `is_focused` flag of this video. + /// + /// # Note + /// It will only actually add operations, if the `is_focused` flag is different. + /// + /// # Panics + /// If the status is not `Cached`. + pub(crate) fn set_focused(&mut self, new_is_focused: bool, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { is_focused, .. } = &mut self.status { + if *is_focused != new_is_focused { + *is_focused = new_is_focused; + + ops.push(Operation::SetIsFocused { + video: self.extractor_hash, + is_focused: new_is_focused, + }); + } + } else { + unreachable!("Can only change `is_focused` on a Cached video."); + } + } + + /// Set the status of this video. + /// + /// # Note + /// This will not actually add any operations, if the new status equals the old one. + pub(crate) fn set_status(&mut self, status: VideoStatus, ops: &mut Operations<Operation>) { + if self.status != status { + status.clone_into(&mut self.status); + + ops.push(Operation::SetStatus { + video: self.extractor_hash, + status, + }); + } + } + + /// Set the priority of this video. + /// + /// # Note + /// This will not actually add any operations, if the new priority equals the old one. + pub(crate) fn set_priority(&mut self, priority: Priority, ops: &mut Operations<Operation>) { + if self.priority != priority { + self.priority = priority; + + ops.push(Operation::SetPriority { + video: self.extractor_hash, + priority, + }); + } + } + + /// Set the watch progress. + /// + /// # Note + /// This will not actually add any operations, + /// if the new watch progress equals the old one. + pub(crate) fn set_watch_progress( + &mut self, + watch_progress: time::Duration, + ops: &mut Operations<Operation>, + ) { + if self.watch_progress != watch_progress { + self.watch_progress = watch_progress; + + ops.push(Operation::SetWatchProgress { + video: self.extractor_hash, + watch_progress, + }); + } + } + + /// Set the playback speed of this video. + /// + /// # Note + /// This will not actually add any operations, if the new speed equals the old one. + pub(crate) fn set_playback_speed( + &mut self, + playback_speed: f64, + ops: &mut Operations<Operation>, + ) { + if self.playback_speed != Some(playback_speed) { + self.playback_speed = Some(playback_speed); + + ops.push(Operation::SetPlaybackSpeed { + video: self.extractor_hash, + playback_speed, + }); + } + } + + /// Set the subtitle langs of this video. + /// + /// # Note + /// This will not actually add any operations, if the new langs equal the old one. + pub(crate) fn set_subtitle_langs( + &mut self, + subtitle_langs: String, + ops: &mut Operations<Operation>, + ) { + if self.subtitle_langs.as_ref() != Some(&subtitle_langs) { + self.subtitle_langs = Some(subtitle_langs.clone()); + + ops.push(Operation::SetSubtitleLangs { + video: self.extractor_hash, + subtitle_langs, + }); + } + } + + /// Mark this video watched. + /// This will both set the status to `Watched` and the `cache_path` to Null. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) fn set_watched(&mut self, ops: &mut Operations<Operation>) { + self.remove_download_path(ops); + self.set_status(VideoStatus::Watched, ops); + } +} diff --git a/crates/yt/src/storage/db/mod.rs b/crates/yt/src/storage/db/mod.rs new file mode 100644 index 0000000..926bab0 --- /dev/null +++ b/crates/yt/src/storage/db/mod.rs @@ -0,0 +1,18 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod get; +pub(crate) mod insert; + +pub(crate) mod extractor_hash; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod txn_log; +pub(crate) mod video; diff --git a/crates/yt/src/storage/db/playlist/mod.rs b/crates/yt/src/storage/db/playlist/mod.rs new file mode 100644 index 0000000..7366e8e --- /dev/null +++ b/crates/yt/src/storage/db/playlist/mod.rs @@ -0,0 +1,59 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::ops::Add; + +use crate::storage::db::video::Video; + +/// Zero-based index into the internal playlist. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PlaylistIndex(usize); + +impl From<PlaylistIndex> for usize { + fn from(value: PlaylistIndex) -> Self { + value.0 + } +} + +impl From<usize> for PlaylistIndex { + fn from(value: usize) -> Self { + Self(value) + } +} + +impl Add<usize> for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl Add for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +/// A representation of the internal Playlist +#[derive(Debug)] +pub(crate) struct Playlist { + pub(crate) videos: Vec<Video>, +} + +impl Playlist { + /// Returns the number of videos in the playlist + #[must_use] + pub(crate) fn len(&self) -> usize { + self.videos.len() + } +} diff --git a/crates/yt/src/storage/db/subscription.rs b/crates/yt/src/storage/db/subscription.rs new file mode 100644 index 0000000..39385b9 --- /dev/null +++ b/crates/yt/src/storage/db/subscription.rs @@ -0,0 +1,58 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::collections::HashMap; + +use anyhow::Result; +use log::debug; +use serde::{Deserialize, Serialize}; +use url::Url; +use yt_dlp::{json_cast, json_try_get, options::YoutubeDLOptions}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct Subscription { + /// The human readable name of this subscription + pub(crate) name: String, + + /// The URL this subscription subscribes to + pub(crate) url: Url, + + pub(crate) is_active: bool, +} + +impl Subscription { + #[must_use] + pub(crate) fn new(name: String, url: Url, is_active: bool) -> Self { + Self { + name, + url, + is_active, + } + } +} + +#[derive(Default, Debug)] +pub(crate) struct Subscriptions(pub(crate) HashMap<String, Subscription>); + +/// Check whether an URL could be used as a subscription URL +pub(crate) async fn check_url(url: Url) -> Result<bool> { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + debug!("{info:#?}"); + + Ok(json_try_get!(info, "_type", as_str) == Some("playlist")) +} diff --git a/crates/yt/src/storage/db/txn_log.rs b/crates/yt/src/storage/db/txn_log.rs new file mode 100644 index 0000000..64884b0 --- /dev/null +++ b/crates/yt/src/storage/db/txn_log.rs @@ -0,0 +1,24 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::storage::db::{insert::Committable, video::TimeStamp}; + +pub(crate) struct TxnLog<O: Committable> { + inner: Vec<(TimeStamp, O)>, +} + +impl<O: Committable> TxnLog<O> { + pub(crate) fn new(inner: Vec<(TimeStamp, O)>) -> Self { + Self { inner } + } + pub(crate) fn inner(&self) -> &[(TimeStamp, O)] { + &self.inner + } +} diff --git a/crates/yt/src/comments/display.rs b/crates/yt/src/storage/db/video/comments/display.rs index 6166b2b..c372603 100644 --- a/crates/yt/src/comments/display.rs +++ b/crates/yt/src/storage/db/video/comments/display.rs @@ -1,6 +1,5 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> // Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // @@ -13,27 +12,22 @@ use std::fmt::Write; use chrono::{Local, TimeZone}; use chrono_humanize::{Accuracy, HumanTime, Tense}; +use colors::{Colorize, IntoCanvas}; -use crate::comments::comment::CommentExt; - -use super::comment::Comments; +use crate::{ + output::format_text, + storage::db::video::comments::{Comment, Comments}, +}; impl Comments { - pub fn render(&self, color: bool) -> String { - self.render_help(color).expect("This should never fail.") + pub(crate) fn render(&self, use_color: bool) -> String { + self.render_help(use_color) + .expect("This should never fail.") } - fn render_help(&self, color: bool) -> Result<String, std::fmt::Error> { - macro_rules! c { - ($color_str:expr, $write:ident, $color:expr) => { - if $color { - $write.write_str(concat!("\x1b[", $color_str, "m"))? - } - }; - } - + fn render_help(&self, use_color: bool) -> Result<String, std::fmt::Error> { fn format( - comment: &CommentExt, + comment: &Comment, f: &mut String, ident_count: u32, color: bool, @@ -43,14 +37,16 @@ impl Comments { f.write_str(ident)?; - if value.author_is_uploader { - c!("91;1", f, color); - } else { - c!("35", f, color); - } + write!( + f, + "{}", + if value.author_is_uploader { + (&value.author).bold().bright_red().render(color) + } else { + (&value.author).purple().render(color) + } + )?; - f.write_str(&value.author)?; - c!("0", f, color); if value.edited || value.is_favorited { f.write_str("[")?; if value.edited { @@ -65,7 +61,6 @@ impl Comments { f.write_str("]")?; } - c!("36;1", f, color); write!( f, " {}", @@ -76,17 +71,31 @@ impl Comments { .expect("This should be valid") ) .to_text_en(Accuracy::Rough, Tense::Past) + .bold() + .cyan() + .render(color) )?; - c!("0", f, color); - // c!("31;1", f); - // f.write_fmt(format_args!(" [{}]", comment.value.like_count))?; - // c!("0", f); + write!( + f, + " [{}]", + comment.value.like_count.bold().red().render(color) + )?; f.write_str(":\n")?; f.write_str(ident)?; - f.write_str(&value.text.replace('\n', &format!("\n{ident}")))?; + f.write_str( + &format_text( + value.text.trim(), + Some( + termsize::get().map_or(90, |ts| ts.cols) + - u16::try_from(ident_count).expect("Should never overflow"), + ), + ) + .trim() + .replace('\n', &format!("\n{ident}")), + )?; f.write_str("\n")?; if comment.replies.is_empty() { @@ -105,12 +114,12 @@ impl Comments { let mut f = String::new(); - if !&self.vec.is_empty() { - let mut children = self.vec.clone(); + if !&self.inner.is_empty() { + let mut children = self.inner.clone(); children.sort_by(|a, b| b.value.like_count.cmp(&a.value.like_count)); for child in children { - format(&child, &mut f, 0, color)?; + format(&child, &mut f, 0, use_color)?; } } Ok(f) diff --git a/crates/yt/src/storage/db/video/comments/mod.rs b/crates/yt/src/storage/db/video/comments/mod.rs new file mode 100644 index 0000000..41a03be --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/mod.rs @@ -0,0 +1,202 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::mem; + +use regex::{Captures, Regex}; + +use crate::storage::db::video::comments::raw::{Parent, RawComment}; + +pub(crate) mod display; +pub(crate) mod raw; + +#[cfg(test)] +mod tests; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Comment { + value: RawComment, + replies: Vec<Self>, +} + +#[derive(Debug, Default, PartialEq)] +pub(crate) struct Comments { + inner: Vec<Comment>, +} + +impl Comments { + pub(crate) fn from_raw(raw: Vec<RawComment>) -> Self { + let mut me = Self::default(); + + // Apply the parent -> child mapping yt provides us with. + for raw_comment in raw { + if let Parent::Id(id) = &raw_comment.parent { + me.insert(&(id.clone()), Comment::from(raw_comment)); + } else { + me.inner.push(Comment::from(raw_comment)); + } + } + + { + // Sort the final comments chronologically. + // This ensures that replies are matched with the comment they actually replied to and + // not a later comment from the same author. + for comment in &mut me.inner { + comment + .replies + .sort_by_key(|comment| comment.value.timestamp); + + for reply in &comment.replies { + assert!(reply.replies.is_empty()); + } + } + } + + { + let find_reply_indicator = + Regex::new(r"\u{200b}?(@[^\t\s]+)\u{200b}?").expect("This is hardcoded"); + + // Try to re-construct the replies for the reply comments. + for comment in &mut me.inner { + let previous_replies = mem::take(&mut comment.replies); + + let mut reply_tree = Comments::default(); + + for reply in previous_replies { + // We try to reconstruct the parent child relation ship by looking (naively) + // for a reply indicator. Currently, this is just the `@<some_name>`, as yt + // seems to insert that by default if you press `reply-to` in their clients. + // + // This follows these steps: + // - Does this reply have a “reply indicator”? + // - If yes, try to resolve the indicator. + // - If it is resolvable, add this reply to the [`Comment`] it resolved to. + // - If not, keep the comment as reply. + + if let Some(reply_indicator_matches) = + find_reply_indicator.captures(&reply.value.text.clone()) + { + // We found a reply indicator. + // First we traverse the current `reply_tree` in reversed order to find a + // match, than we check if the reply indicator matches the reply tree root + // and afterward we declare it unmatching and add it as toplevel. + + let reply_target_author = reply_indicator_matches + .get(1) + .expect("This should also exist") + .as_str(); + + if let Some(parent) = reply_tree.find_author_mut(reply_target_author) { + parent + .replies + .push(comment_from_reply(reply, &reply_indicator_matches)); + } else if comment.value.author == reply_target_author { + reply_tree + .add_toplevel(comment_from_reply(reply, &reply_indicator_matches)); + } else { + eprintln!( + "Failed to find a parent for ('{}') both directly \ + and via replies! The reply text was:\n'{}'\n", + reply_target_author, reply.value.text + ); + reply_tree.add_toplevel(reply); + } + } else { + // The comment text did not contain a reply indicator, so add it as + // toplevel. + reply_tree.add_toplevel(reply); + } + } + + comment.replies = reply_tree.inner; + } + } + + me + } + + fn add_toplevel(&mut self, value: Comment) { + self.inner.push(value); + } + + fn insert(&mut self, id: &str, value: Comment) { + let parent = self + .inner + .iter_mut() + .find(|c| c.value.id.id == id) + .expect("One of these should exist"); + + parent.replies.push(value); + } + + fn find_author_mut(&mut self, reply_target_author: &str) -> Option<&mut Comment> { + fn perform_check<'a>( + comment: &'a mut Comment, + reply_target_author: &str, + ) -> Option<&'a mut Comment> { + // TODO(@bpeetz): This is a workaround until rust has lexiographic lifetime support. <2025-07-18> + fn find_in_replies<'a>( + comment: &'a mut Comment, + reply_target_author: &str, + ) -> Option<&'a mut Comment> { + comment + .replies + .iter_mut() + .rev() + .find_map(|reply: &mut Comment| perform_check(reply, reply_target_author)) + } + let comment_author_matches_target = comment.value.author == reply_target_author; + + match find_in_replies(comment, reply_target_author) { + Some(_) => Some( + // PERFORMANCE(@bpeetz): We should not need to run this code twice. <2025-07-18> + find_in_replies(comment, reply_target_author) + .expect("We already had a Some result for this."), + ), + None if comment_author_matches_target => Some(comment), + None => None, + } + } + + for comment in self.inner.iter_mut().rev() { + if let Some(output) = perform_check(comment, reply_target_author) { + return Some(output); + } + } + + None + } +} +fn comment_from_reply(reply: Comment, reply_indicator_matches: &Captures<'_>) -> Comment { + Comment::from(RawComment { + text: { + // Remove the `@<some_name>` for the comment text. + let full_match = reply_indicator_matches + .get(0) + .expect("This will always exist"); + + let text = reply.value.text[0..full_match.start()].to_owned() + + &reply.value.text[full_match.end()..]; + + text.trim_matches(|c: char| c == '\u{200b}' || c == '\u{2060}' || c.is_whitespace()) + .to_owned() + }, + ..reply.value + }) +} + +impl From<RawComment> for Comment { + fn from(value: RawComment) -> Self { + Self { + value, + replies: vec![], + } + } +} diff --git a/crates/yt/src/storage/db/video/comments/raw.rs b/crates/yt/src/storage/db/video/comments/raw.rs new file mode 100644 index 0000000..3b7f40f --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/raw.rs @@ -0,0 +1,87 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use serde::{Deserialize, Deserializer}; +use url::Url; + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub(crate) struct Id { + pub(crate) id: String, +} +impl From<String> for Id { + fn from(value: String) -> Self { + Self { + // Take the last element if the string is split with dots, otherwise take the full id + id: value.split('.').next_back().unwrap_or(&value).to_owned(), + } + } +} + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub(crate) enum Parent { + Root, + Id(String), +} + +impl From<String> for Parent { + fn from(value: String) -> Self { + if value == "root" { + Self::Root + } else { + Self::Id(value) + } + } +} + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[allow(clippy::struct_excessive_bools)] +pub(crate) struct RawComment { + pub(crate) id: Id, + pub(crate) text: String, + #[serde(default = "zero")] + pub(crate) like_count: u32, + pub(crate) is_pinned: bool, + pub(crate) author_id: String, + #[serde(default = "unknown")] + pub(crate) author: String, + pub(crate) author_is_verified: bool, + pub(crate) author_thumbnail: Url, + pub(crate) parent: Parent, + #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] + pub(crate) edited: bool, + // Can't also be deserialized, as it's already used in 'edited' + // _time_text: String, + pub(crate) timestamp: i64, + pub(crate) author_url: Option<Url>, + pub(crate) author_is_uploader: bool, + pub(crate) is_favorited: bool, +} + +fn unknown() -> String { + "<Unknown>".to_string() +} +fn zero() -> u32 { + 0 +} +fn edited_from_time_text<'de, D>(d: D) -> Result<bool, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(d)?; + if s.contains(" (edited)") { + Ok(true) + } else { + Ok(false) + } +} diff --git a/crates/yt/src/storage/db/video/comments/tests.rs b/crates/yt/src/storage/db/video/comments/tests.rs new file mode 100644 index 0000000..03e3597 --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/tests.rs @@ -0,0 +1,249 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use pretty_assertions::assert_eq; +use url::Url; + +use crate::storage::db::video::comments::{ + Comment, Comments, RawComment, + raw::{Id, Parent}, +}; + +/// Generate both an [`expected`] and an [`input`] value from an expected comment expression. +macro_rules! mk_comments { + () => {{ + let input: Vec<RawComment> = vec![]; + let expected: Comments = Comments { + inner: vec![], + }; + + (input, expected) + }}; + + ( + $( + parent: $parent:expr, $actual_parent:ident, + ( + @ $name:ident : $comment:literal + $( + $reply_chain:tt + )* + ) + )+ + ) => {{ + let (nested_input, _) = mk_comments!( + $( + $( + parent: $parent, $name, + $reply_chain + )* + )+ + ); + + let mut input: Vec<RawComment> = vec![ + $( + mk_comments!(@to_raw input $name $comment $parent, $actual_parent) + ),+ + ]; + input.extend(nested_input); + + let expected: Comments = Comments { + inner: vec![ + $( + Comment { + value: mk_comments!(@to_raw expected $name $comment $parent, $actual_parent), + replies: { + let (_, nested_expected) = mk_comments!( + $( + parent: $parent, $name, + $reply_chain + )* + ); + + nested_expected.inner + }, + } + ),+ + ] + }; + + (input, expected) + }}; + ( + $( + ( + @ $name:ident : $comment:literal + $( + $reply_chain:tt + )* + ) + )+ + ) => {{ + let (nested_input, _) = mk_comments!( + $( + $( + parent: mk_comments!(@mk_id $name $comment), $name, + $reply_chain + )* + )+ + ); + + let mut input: Vec<RawComment> = vec![ + $( + mk_comments!(@to_raw input $name $comment) + ),+ + ]; + input.extend(nested_input); + + let expected: Comments = Comments { + inner: vec![ + $( + Comment { + value: mk_comments!(@to_raw expected $name $comment), + replies: { + let (_, nested_expected) = mk_comments!( + $( + parent: mk_comments!(@mk_id $name $comment), $name, + $reply_chain + )* + ); + + nested_expected.inner + }, + } + ),+ + ] + }; + + (input, expected) + }}; + + (@mk_id $name:ident $comment:literal) => {{ + use std::hash::{Hash, Hasher}; + + let input = format!("{}{}", stringify!($name), $comment); + + let mut digest = std::hash::DefaultHasher::new(); + input.hash(&mut digest); + Id { id: digest.finish().to_string() } + }}; + + (@to_raw $state:ident $name:ident $comment:literal $($parent:expr, $actual_parent:ident)?) => { + RawComment { + id: mk_comments!(@mk_id $name $comment), + text: mk_comments!(@mk_text $state $comment $(, $actual_parent)?), + like_count: 0, + is_pinned: false, + author_id: stringify!($name).to_owned(), + author: format!("@{}", stringify!($name)), + author_is_verified: false, + author_thumbnail: Url::from_file_path("/dev/null").unwrap(), + parent: mk_comments!(@mk_parent $($parent)?), + edited: false, + timestamp: 0, + author_url: None, + author_is_uploader: false, + is_favorited: false, + } + }; + + (@mk_parent) => { + Parent::Root + }; + (@mk_parent $parent:expr) => { + Parent::Id($parent.id) + }; + + (@mk_text input $text:expr) => { + $text.to_owned() + }; + (@mk_text input $text:expr, $actual_parent:ident) => { + format!("@{} {}", stringify!($actual_parent), $text) + }; + (@mk_text expected $text:expr $(, $_:tt)?) => { + $text.to_owned() + }; +} + +#[test] +fn test_comments_toplevel() { + let (input, expected) = mk_comments!( + (@kant: "I think, that using the results of an action to determine morality is flawed.") + (@hume: "I think, that we should use our feeling for morality more.") + (@lock: "I think, that we should rely on the sum of happiness caused by an action to determine it's morality.") + ); + + assert_eq!(Comments::from_raw(input), expected); +} + +#[test] +fn test_comments_replies_1_level() { + let (input, expected) = mk_comments!( + (@hume: "I think, that we should use our feeling for morality more." + (@kant: "This is so wrong! I shall now dedicate my next 7? years to writing books that prove this.") + (@lock: "It feels not very applicable, no? We should focus on something that can be used in the court of law!")) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_replies_2_levels() { + let (input, expected) = mk_comments!( + (@singer: "We perform medical studies on animals; Children have lower or similar mental ability as these animals.." + (@singer: "Therefore, we should perform these studies on children instead, if we were to follow our own principals" + (@james: "This is ridiculous! I will not entertain this thought.") + (@singer: "Although one could also use this argument to argue for abortion _after_ birth."))) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_replies_3_levels() { + let (input, expected) = mk_comments!( + (@singer: "We perform medical studies on animals; Children have lower or similar mental ability as these animals.." + (@singer: "Therefore, we should perform these studies on children instead, if we were to follow our own principals" + (@james: "This is ridiculous! I will not entertain this thought." + (@singer: "You know that I am not actually suggesting that? This is but a way to critizise the society")) + (@singer: "Although one could also use this argument to argue for abortion _after_ birth."))) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_sub_answer_selection() { + let (input, expected) = mk_comments!( + (@coffeewolfproductions9113: "I mean, brothels and sex workers in of themselves are not a bad thing." + (@aikikaname6508: "probably not so much in the 50s, pre contraception") + (@as_ri1mb: "it’s an incredibly sad, degrading line of work, often resulting in self loathing and self-deletion." + (@coffeewolfproductions9113: "Are you speaking from experience?" + (@as_ri1mb: "what an immature response, as expected." + (@coffeewolfproductions9113: "I literally just asked if you were talking from experience."))))) + + ); + + eprintln!("{}", expected.render(true)); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} diff --git a/crates/yt/src/storage/video_database/mod.rs b/crates/yt/src/storage/db/video/mod.rs index 74d09f0..deeb82c 100644 --- a/crates/yt/src/storage/video_database/mod.rs +++ b/crates/yt/src/storage/db/video/mod.rs @@ -1,6 +1,5 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> // Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // @@ -9,55 +8,108 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use std::{ - fmt::{Display, Write}, - path::PathBuf, - time::Duration, -}; +use std::{fmt::Display, path::PathBuf, time::Duration}; use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; use url::Url; -use crate::{ - app::App, select::selection_file::duration::MaybeDuration, - storage::video_database::extractor_hash::ExtractorHash, -}; +use crate::{select::duration::MaybeDuration, storage::db::extractor_hash::ExtractorHash}; -pub mod downloader; -pub mod extractor_hash; -pub mod get; -pub mod notify; -pub mod set; +pub(crate) mod comments; + +macro_rules! video_from_record { + ($record:expr) => { + $crate::storage::db::video::Video { + description: $record.description.clone(), + duration: $crate::select::duration::MaybeDuration::from_maybe_secs_f64( + $record.duration, + ), + extractor_hash: $crate::storage::db::extractor_hash::ExtractorHash::from_hash( + $record + .extractor_hash + .parse() + .expect("The db hash should be a valid blake3 hash"), + ), + last_status_change: $crate::storage::db::video::TimeStamp::from_secs( + $record.last_status_change, + ), + parent_subscription_name: $record.parent_subscription_name.clone(), + publish_date: $record + .publish_date + .map(|pd| $crate::storage::db::video::TimeStamp::from_secs(pd)), + status: { + let marker = + $crate::storage::db::video::VideoStatusMarker::from_db_integer($record.status); + let optional = if let Some(cache_path) = &$record.cache_path { + Some(( + std::path::PathBuf::from(cache_path), + if $record.is_focused == Some(1) { + true + } else { + false + }, + )) + } else { + None + }; + $crate::storage::db::video::VideoStatus::from_marker(marker, optional) + }, + thumbnail_url: if let Some(url) = &$record.thumbnail_url { + Some(url::Url::parse(url).expect("Parsing this as url should always work")) + } else { + None + }, + title: $record.title.clone(), + url: url::Url::parse(&$record.url).expect("Parsing this as url should always work"), + priority: $crate::storage::db::video::Priority::from($record.priority), + watch_progress: std::time::Duration::from_secs( + u64::try_from($record.watch_progress).expect("The record is positive i64"), + ), + subtitle_langs: $record.subtitle_langs.clone(), + playback_speed: $record.playback_speed, + } + }; +} +pub(crate) use video_from_record; #[derive(Debug, Clone)] -pub struct Video { - pub description: Option<String>, - pub duration: MaybeDuration, - pub extractor_hash: ExtractorHash, - pub last_status_change: TimeStamp, +pub(crate) struct Video { + pub(crate) description: Option<String>, + pub(crate) duration: MaybeDuration, + pub(crate) extractor_hash: ExtractorHash, + pub(crate) last_status_change: TimeStamp, /// The associated subscription this video was fetched from (null, when the video was `add`ed) - pub parent_subscription_name: Option<String>, - pub priority: Priority, - pub publish_date: Option<TimeStamp>, - pub status: VideoStatus, - pub thumbnail_url: Option<Url>, - pub title: String, - pub url: Url, + pub(crate) parent_subscription_name: Option<String>, + pub(crate) priority: Priority, + pub(crate) publish_date: Option<TimeStamp>, + pub(crate) status: VideoStatus, + pub(crate) thumbnail_url: Option<Url>, + pub(crate) title: String, + pub(crate) url: Url, /// The seconds the user has already watched the video - pub watch_progress: Duration, + pub(crate) watch_progress: Duration, + + /// Which subtitles to include, when downloading this video. + /// In the form of `lang1,lang2,lang3` (e.g. `en,de,sv`) + pub(crate) subtitle_langs: Option<String>, + + /// The playback speed to use, when watching this video. + /// Value is in percent, so 1 is 100%, 2.7 is 270%, and so on. + pub(crate) playback_speed: Option<f64>, } /// The priority of a [`Video`]. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct Priority { +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct Priority { value: i64, } impl Priority { /// Return the underlying value to insert that into the database #[must_use] - pub fn as_db_integer(&self) -> i64 { + pub(crate) fn as_db_integer(self) -> i64 { self.value } } @@ -74,25 +126,25 @@ impl Display for Priority { /// An UNIX time stamp. #[derive(Debug, Default, Clone, Copy)] -pub struct TimeStamp { +pub(crate) struct TimeStamp { value: i64, } impl TimeStamp { /// Return the seconds since the UNIX epoch for this [`TimeStamp`]. #[must_use] - pub fn as_secs(&self) -> i64 { + pub(crate) fn as_secs(self) -> i64 { self.value } /// Construct a [`TimeStamp`] from a count of seconds since the UNIX epoch. #[must_use] - pub fn from_secs(value: i64) -> Self { + pub(crate) fn from_secs(value: i64) -> Self { Self { value } } /// Construct a [`TimeStamp`] from the current time. #[must_use] - pub fn from_now() -> Self { + pub(crate) fn from_now() -> Self { Self { value: Utc::now().timestamp(), } @@ -107,49 +159,6 @@ impl Display for TimeStamp { } } -#[derive(Debug)] -pub struct VideoOptions { - pub yt_dlp: YtDlpOptions, - pub mpv: MpvOptions, -} -impl VideoOptions { - pub(crate) fn new(subtitle_langs: String, playback_speed: f64) -> Self { - let yt_dlp = YtDlpOptions { subtitle_langs }; - let mpv = MpvOptions { playback_speed }; - Self { yt_dlp, mpv } - } - - /// This will write out the options that are different from the defaults. - /// Beware, that this does not set the priority. - #[must_use] - pub fn to_cli_flags(self, app: &App) -> String { - let mut f = String::new(); - - if (self.mpv.playback_speed - app.config.select.playback_speed).abs() > f64::EPSILON { - write!(f, " --speed '{}'", self.mpv.playback_speed).expect("Works"); - } - if self.yt_dlp.subtitle_langs != app.config.select.subtitle_langs { - write!(f, " --subtitle-langs '{}'", self.yt_dlp.subtitle_langs).expect("Works"); - } - - f.trim().to_owned() - } -} - -#[derive(Debug, Clone, Copy)] -/// Additionally settings passed to mpv on watch -pub struct MpvOptions { - /// The playback speed. (1 is 100%, 2.7 is 270%, and so on) - pub playback_speed: f64, -} - -#[derive(Debug)] -/// Additionally configuration options, passed to yt-dlp on download -pub struct YtDlpOptions { - /// In the form of `lang1,lang2,lang3` (e.g. `en,de,sv`) - pub subtitle_langs: String, -} - /// # Video Lifetime (words in <brackets> are commands): /// <Pick> /// / \ @@ -158,8 +167,8 @@ pub struct YtDlpOptions { /// Cache // yt cache /// | /// Watched // yt watch -#[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] -pub enum VideoStatus { +#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub(crate) enum VideoStatus { #[default] Pick, @@ -186,7 +195,10 @@ impl VideoStatus { /// # Panics /// Only if internal expectations fail. #[must_use] - pub fn from_marker(marker: VideoStatusMarker, optional: Option<(PathBuf, bool)>) -> Self { + pub(crate) fn from_marker( + marker: VideoStatusMarker, + optional: Option<(PathBuf, bool)>, + ) -> Self { match marker { VideoStatusMarker::Pick => Self::Pick, VideoStatusMarker::Watch => Self::Watch, @@ -204,26 +216,9 @@ impl VideoStatus { } } - /// Turn the [`VideoStatus`] to its internal parts. This is only really useful for the database - /// functions. - #[must_use] - pub fn to_parts_for_db(self) -> (VideoStatusMarker, Option<(PathBuf, bool)>) { - match self { - VideoStatus::Pick => (VideoStatusMarker::Pick, None), - VideoStatus::Watch => (VideoStatusMarker::Watch, None), - VideoStatus::Cached { - cache_path, - is_focused, - } => (VideoStatusMarker::Cached, Some((cache_path, is_focused))), - VideoStatus::Watched => (VideoStatusMarker::Watched, None), - VideoStatus::Drop => (VideoStatusMarker::Drop, None), - VideoStatus::Dropped => (VideoStatusMarker::Dropped, None), - } - } - /// Return the associated [`VideoStatusMarker`] for this [`VideoStatus`]. #[must_use] - pub fn as_marker(&self) -> VideoStatusMarker { + pub(crate) fn as_marker(&self) -> VideoStatusMarker { match self { VideoStatus::Pick => VideoStatusMarker::Pick, VideoStatus::Watch => VideoStatusMarker::Watch, @@ -237,7 +232,7 @@ impl VideoStatus { /// Unit only variant of [`VideoStatus`] #[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] -pub enum VideoStatusMarker { +pub(crate) enum VideoStatusMarker { #[default] Pick, @@ -255,7 +250,7 @@ pub enum VideoStatusMarker { } impl VideoStatusMarker { - pub const ALL: &'static [Self; 6] = &[ + pub(crate) const ALL: &'static [Self; 6] = &[ Self::Pick, // Self::Watch, @@ -267,7 +262,7 @@ impl VideoStatusMarker { ]; #[must_use] - pub fn as_command(&self) -> &str { + pub(crate) fn as_command(&self) -> &str { // NOTE: Keep the serialize able variants synced with the main `select` function <2024-06-14> // Also try to ensure, that the strings have the same length match self { @@ -281,7 +276,7 @@ impl VideoStatusMarker { } #[must_use] - pub fn as_db_integer(&self) -> i64 { + pub(crate) fn as_db_integer(self) -> i64 { // These numbers should not change their mapping! // Oh, and keep them in sync with the SQLite check constraint. match self { @@ -296,7 +291,7 @@ impl VideoStatusMarker { } } #[must_use] - pub fn from_db_integer(num: i64) -> Self { + pub(crate) fn from_db_integer(num: i64) -> Self { match num { 0 => Self::Pick, @@ -314,7 +309,7 @@ impl VideoStatusMarker { } #[must_use] - pub fn as_str(&self) -> &'static str { + pub(crate) fn as_str(self) -> &'static str { match self { Self::Pick => "Pick", diff --git a/crates/yt/src/storage/migrate/mod.rs b/crates/yt/src/storage/migrate/mod.rs index 953d079..c5187ee 100644 --- a/crates/yt/src/storage/migrate/mod.rs +++ b/crates/yt/src/storage/migrate/mod.rs @@ -75,7 +75,7 @@ macro_rules! make_upgrade { } #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] -pub enum DbVersion { +pub(crate) enum DbVersion { /// The database is not yet initialized. Empty, @@ -91,8 +91,17 @@ pub enum DbVersion { /// Introduced: 2025-03-21. Three, + + /// Introduced: 2025-07-05. + Four, + + /// Introduced: 2025-07-20. + Five, + + /// Introduced: 2025-08-26. + Six, } -const CURRENT_VERSION: DbVersion = DbVersion::Three; +const CURRENT_VERSION: DbVersion = DbVersion::Six; async fn add_error_context( function: impl Future<Output = Result<()>>, @@ -143,6 +152,9 @@ impl DbVersion { DbVersion::One => 1, DbVersion::Two => 2, DbVersion::Three => 3, + DbVersion::Four => 4, + DbVersion::Five => 5, + DbVersion::Six => 6, DbVersion::Empty => unreachable!("A empty version does not have an associated integer"), } @@ -154,11 +166,17 @@ impl DbVersion { (1, "yt") => Ok(DbVersion::One), (2, "yt") => Ok(DbVersion::Two), (3, "yt") => Ok(DbVersion::Three), + (4, "yt") => Ok(DbVersion::Four), + (5, "yt") => Ok(DbVersion::Five), + (6, "yt") => Ok(DbVersion::Six), (0, other) => bail!("Db version is Zero, but got unknown namespace: '{other}'"), (1, other) => bail!("Db version is One, but got unknown namespace: '{other}'"), (2, other) => bail!("Db version is Two, but got unknown namespace: '{other}'"), (3, other) => bail!("Db version is Three, but got unknown namespace: '{other}'"), + (4, other) => bail!("Db version is Four, but got unknown namespace: '{other}'"), + (5, other) => bail!("Db version is Five, but got unknown namespace: '{other}'"), + (6, other) => bail!("Db version is Six, but got unknown namespace: '{other}'"), (other, "yt") => bail!("Got unkown version for 'yt' namespace: {other}"), (num, nasp) => bail!("Got unkown version number ({num}) and namespace ('{nasp}')"), @@ -188,8 +206,20 @@ impl DbVersion { make_upgrade! {app, Self::Two, Self::Three, "./sql/3_Two_to_Three.sql"} } - // This is the current_version Self::Three => { + make_upgrade! {app, Self::Three, Self::Four, "./sql/4_Three_to_Four.sql"} + } + + Self::Four => { + make_upgrade! {app, Self::Four, Self::Five, "./sql/5_Four_to_Five.sql"} + } + + Self::Five => { + make_upgrade! {app, Self::Five, Self::Six, "./sql/6_Five_to_Six.sql"} + } + + // This is the current_version + Self::Six => { assert_eq!(self, CURRENT_VERSION); assert_eq!(self, get_version(app).await?); Ok(()) @@ -222,9 +252,10 @@ fn get_current_date() -> i64 { /// /// # Panics /// Only if internal assertions fail. -pub async fn get_version(app: &App) -> Result<DbVersion> { +pub(crate) async fn get_version(app: &App) -> Result<DbVersion> { get_version_db(&app.database).await } + /// Return the current database version. /// /// In contrast to the [`get_version`] function, this function does not @@ -232,13 +263,19 @@ pub async fn get_version(app: &App) -> Result<DbVersion> { /// /// # Panics /// Only if internal assertions fail. -pub async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { +pub(crate) async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { let version_table_exists = { let query = query!( - "SELECT 1 as result FROM sqlite_master WHERE type = 'table' AND name = 'version'" + " + SELECT 1 as result + FROM sqlite_master + WHERE type = 'table' + AND name = 'version' + " ) .fetch_optional(pool) .await?; + if let Some(output) = query { assert_eq!(output.result, 1); true @@ -246,13 +283,16 @@ pub async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { false } }; + if !version_table_exists { return Ok(DbVersion::Empty); } let current_version = query!( " - SELECT namespace, number FROM version WHERE valid_to IS NULL; + SELECT namespace, number + FROM version + WHERE valid_to IS NULL; " ) .fetch_one(pool) @@ -262,7 +302,7 @@ pub async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { DbVersion::from_db(current_version.number, current_version.namespace.as_str()) } -pub async fn migrate_db(app: &App) -> Result<()> { +pub(crate) async fn migrate_db(app: &App) -> Result<()> { let current_version = get_version(app) .await .context("Failed to determine initial version")?; diff --git a/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql b/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql new file mode 100644 index 0000000..9c283a1 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql @@ -0,0 +1,24 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +ALTER TABLE videos +ADD COLUMN subtitle_langs TEXT; + +ALTER TABLE videos +ADD COLUMN playback_speed REAL CHECK (playback_speed >= 0); + +UPDATE videos + SET playback_speed = video_options.playback_speed, + subtitle_langs = video_options.subtitle_langs + FROM video_options + WHERE videos.extractor_hash = video_options.extractor_hash; + + +DROP TABLE video_options; diff --git a/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql b/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql new file mode 100644 index 0000000..6c4b7cc --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql @@ -0,0 +1,15 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + + +CREATE TABLE txn_log ( + timestamp INTEGER NOT NULL, + operation TEXT NOT NULL +) STRICT; diff --git a/crates/yt/src/storage/migrate/sql/6_Five_to_Six.sql b/crates/yt/src/storage/migrate/sql/6_Five_to_Six.sql new file mode 100644 index 0000000..6a2cbcc --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/6_Five_to_Six.sql @@ -0,0 +1,12 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +ALTER TABLE subscriptions +ADD COLUMN is_active INTEGER NOT NULL DEFAULT 1 CHECK (is_active IN (0, 1)); diff --git a/crates/yt/src/storage/mod.rs b/crates/yt/src/storage/mod.rs index d352b41..6dcff74 100644 --- a/crates/yt/src/storage/mod.rs +++ b/crates/yt/src/storage/mod.rs @@ -9,6 +9,6 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -pub mod migrate; -pub mod subscriptions; -pub mod video_database; +pub(crate) mod db; +pub(crate) mod migrate; +pub(crate) mod notify; diff --git a/crates/yt/src/storage/video_database/notify.rs b/crates/yt/src/storage/notify.rs index b55c00a..e0ee4e9 100644 --- a/crates/yt/src/storage/video_database/notify.rs +++ b/crates/yt/src/storage/notify.rs @@ -26,7 +26,7 @@ use tokio::task; /// This functions registers a watcher for the database and only returns once a write was /// registered for the database. -pub async fn wait_for_db_write(app: &App) -> Result<()> { +pub(crate) async fn wait_for_db_write(app: &App) -> Result<()> { let db_path: PathBuf = app.config.paths.database_path.clone(); task::spawn_blocking(move || wait_for_db_write_sync(&db_path)).await? } @@ -53,7 +53,7 @@ fn wait_for_db_write_sync(db_path: &Path) -> Result<()> { } /// This functions registers a watcher for the cache path and returns once a file was removed -pub async fn wait_for_cache_reduction(app: &App) -> Result<()> { +pub(crate) async fn wait_for_cache_reduction(app: &App) -> Result<()> { let download_directory: PathBuf = app.config.paths.download_dir.clone(); task::spawn_blocking(move || wait_for_cache_reduction_sync(&download_directory)).await? } diff --git a/crates/yt/src/storage/subscriptions.rs b/crates/yt/src/storage/subscriptions.rs deleted file mode 100644 index 6c0d08a..0000000 --- a/crates/yt/src/storage/subscriptions.rs +++ /dev/null @@ -1,141 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -//! Handle subscriptions - -use std::collections::HashMap; - -use anyhow::Result; -use log::debug; -use sqlx::query; -use url::Url; -use yt_dlp::YoutubeDLOptions; - -use crate::{app::App, unreachable::Unreachable}; - -#[derive(Clone, Debug)] -pub struct Subscription { - /// The human readable name of this subscription - pub name: String, - - /// The URL this subscription subscribes to - pub url: Url, -} - -impl Subscription { - #[must_use] - pub fn new(name: String, url: Url) -> Self { - Self { name, url } - } -} - -/// Check whether an URL could be used as a subscription URL -pub async fn check_url(url: Url) -> Result<bool> { - let yt_dlp = YoutubeDLOptions::new() - .set("playliststart", 1) - .set("playlistend", 10) - .set("noplaylist", false) - .set("extract_flat", "in_playlist") - .build()?; - - let info = yt_dlp.extract_info(&url, false, false)?; - - debug!("{:#?}", info); - - Ok(info.get("_type") == Some(&serde_json::Value::String("Playlist".to_owned()))) -} - -#[derive(Default, Debug)] -pub struct Subscriptions(pub(crate) HashMap<String, Subscription>); - -/// Remove all subscriptions -pub async fn remove_all(app: &App) -> Result<()> { - query!( - " - DELETE FROM subscriptions; - ", - ) - .execute(&app.database) - .await?; - - Ok(()) -} - -/// Get a list of subscriptions -pub async fn get(app: &App) -> Result<Subscriptions> { - let raw_subs = query!( - " - SELECT * - FROM subscriptions; - " - ) - .fetch_all(&app.database) - .await?; - - let subscriptions: HashMap<String, Subscription> = raw_subs - .into_iter() - .map(|sub| { - ( - sub.name.clone(), - Subscription::new( - sub.name, - Url::parse(&sub.url).unreachable("It was an URL, when we inserted it."), - ), - ) - }) - .collect(); - - Ok(Subscriptions(subscriptions)) -} - -pub async fn add_subscription(app: &App, sub: &Subscription) -> Result<()> { - let url = sub.url.to_string(); - - query!( - " - INSERT INTO subscriptions ( - name, - url - ) VALUES (?, ?); - ", - sub.name, - url - ) - .execute(&app.database) - .await?; - - println!("Subscribed to '{}' at '{}'", sub.name, sub.url); - Ok(()) -} - -/// # Panics -/// Only if assertions fail -pub async fn remove_subscription(app: &App, sub: &Subscription) -> Result<()> { - let output = query!( - " - DELETE FROM subscriptions - WHERE name = ? - ", - sub.name, - ) - .execute(&app.database) - .await?; - - assert_eq!( - output.rows_affected(), - 1, - "The remove subscriptino query did effect more (or less) than one row. This is a bug." - ); - - println!("Unsubscribed from '{}' at '{}'", sub.name, sub.url); - - Ok(()) -} diff --git a/crates/yt/src/storage/video_database/downloader.rs b/crates/yt/src/storage/video_database/downloader.rs deleted file mode 100644 index a95081e..0000000 --- a/crates/yt/src/storage/video_database/downloader.rs +++ /dev/null @@ -1,130 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::path::{Path, PathBuf}; - -use anyhow::Result; -use log::debug; -use sqlx::query; - -use crate::{ - app::App, - storage::video_database::{VideoStatus, VideoStatusMarker}, - unreachable::Unreachable, - video_from_record, -}; - -use super::{ExtractorHash, Video}; - -/// Returns to next video which should be downloaded. This respects the priority assigned by select. -/// It does not return videos, which are already cached. -/// -/// # Panics -/// Only if assertions fail. -pub async fn get_next_uncached_video(app: &App) -> Result<Option<Video>> { - let status = VideoStatus::Watch.as_marker().as_db_integer(); - - // NOTE: The ORDER BY statement should be the same as the one in [`get::videos`].<2024-08-22> - let result = query!( - r#" - SELECT * - FROM videos - WHERE status = ? AND cache_path IS NULL - ORDER BY priority DESC, publish_date DESC - LIMIT 1; - "#, - status - ) - .fetch_one(&app.database) - .await; - - if let Err(sqlx::Error::RowNotFound) = result { - Ok(None) - } else { - let base = result?; - - Ok(Some(video_from_record! {base})) - } -} - -/// Update the cached path of a video. Will be set to NULL if the path is None -/// This will also set the status to `Cached` when path is Some, otherwise it set's the status to -/// `Watch`. -pub async fn set_video_cache_path( - app: &App, - video: &ExtractorHash, - path: Option<&Path>, -) -> Result<()> { - if let Some(path) = path { - debug!( - "Setting cache path from '{}' to '{}'", - video.into_short_hash(app).await?, - path.display() - ); - - let path_str = path.display().to_string(); - let extractor_hash = video.hash().to_string(); - let status = VideoStatusMarker::Cached.as_db_integer(); - - query!( - r#" - UPDATE videos - SET cache_path = ?, status = ? - WHERE extractor_hash = ?; - "#, - path_str, - status, - extractor_hash - ) - .execute(&app.database) - .await?; - - Ok(()) - } else { - debug!( - "Setting cache path from '{}' to NULL", - video.into_short_hash(app).await?, - ); - - let extractor_hash = video.hash().to_string(); - let status = VideoStatus::Watch.as_marker().as_db_integer(); - - query!( - r#" - UPDATE videos - SET cache_path = NULL, status = ? - WHERE extractor_hash = ?; - "#, - status, - extractor_hash - ) - .execute(&app.database) - .await?; - - Ok(()) - } -} - -/// Returns the number of cached videos -pub async fn get_allocated_cache(app: &App) -> Result<u32> { - let count = query!( - r#" - SELECT COUNT(cache_path) as count - FROM videos - WHERE cache_path IS NOT NULL; -"#, - ) - .fetch_one(&app.database) - .await?; - - Ok(u32::try_from(count.count) - .unreachable("The value should be strictly positive (and bolow `u32::Max`)")) -} diff --git a/crates/yt/src/storage/video_database/extractor_hash.rs b/crates/yt/src/storage/video_database/extractor_hash.rs deleted file mode 100644 index df545d7..0000000 --- a/crates/yt/src/storage/video_database/extractor_hash.rs +++ /dev/null @@ -1,163 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{collections::HashSet, fmt::Display, str::FromStr}; - -use anyhow::{Context, Result, bail}; -use blake3::Hash; -use log::debug; -use tokio::sync::OnceCell; - -use crate::{app::App, storage::video_database::get::get_all_hashes, unreachable::Unreachable}; - -static EXTRACTOR_HASH_LENGTH: OnceCell<usize> = OnceCell::const_new(); - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] -pub struct ExtractorHash { - hash: Hash, -} - -impl Display for ExtractorHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.hash.fmt(f) - } -} - -#[derive(Debug, Clone)] -pub struct ShortHash(String); - -impl Display for ShortHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -#[derive(Debug, Clone)] -#[allow(clippy::module_name_repetitions)] -pub struct LazyExtractorHash { - value: ShortHash, -} - -impl FromStr for LazyExtractorHash { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { - // perform some cheap validation - if s.len() > 64 { - bail!("A hash can only contain 64 bytes!"); - } - - Ok(Self { - value: ShortHash(s.to_owned()), - }) - } -} - -impl LazyExtractorHash { - /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] - pub async fn realize(self, app: &App) -> Result<ExtractorHash> { - ExtractorHash::from_short_hash(app, &self.value).await - } -} - -impl ExtractorHash { - #[must_use] - pub fn from_hash(hash: Hash) -> Self { - Self { hash } - } - pub async fn from_short_hash(app: &App, s: &ShortHash) -> Result<Self> { - Ok(Self { - hash: Self::short_hash_to_full_hash(app, s).await?, - }) - } - - #[must_use] - pub fn hash(&self) -> &Hash { - &self.hash - } - - pub async fn into_short_hash(&self, app: &App) -> Result<ShortHash> { - let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { - *needed_chars - } else { - let needed_chars = self - .get_needed_char_len(app) - .await - .context("Failed to calculate needed char length")?; - EXTRACTOR_HASH_LENGTH.set(needed_chars).unreachable( - "This should work at this stage, as we checked above that it is empty.", - ); - - needed_chars - }; - - Ok(ShortHash( - self.hash() - .to_hex() - .chars() - .take(needed_chars) - .collect::<String>(), - )) - } - - async fn short_hash_to_full_hash(app: &App, s: &ShortHash) -> Result<Hash> { - let all_hashes = get_all_hashes(app) - .await - .context("Failed to fetch all extractor -hashesh from database")?; - - let needed_chars = s.0.len(); - - for hash in all_hashes { - if hash.to_hex()[..needed_chars] == s.0 { - return Ok(hash); - } - } - - bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); - } - - async fn get_needed_char_len(&self, app: &App) -> Result<usize> { - debug!("Calculating the needed hash char length"); - let all_hashes = get_all_hashes(app) - .await - .context("Failed to fetch all extractor -hashesh from database")?; - - let all_char_vec_hashes = all_hashes - .into_iter() - .map(|hash| hash.to_hex().chars().collect::<Vec<char>>()) - .collect::<Vec<Vec<_>>>(); - - // This value should be updated later, if not rust will panic in the assertion. - let mut needed_chars: usize = 1000; - 'outer: for i in 1..64 { - let i_chars: Vec<String> = all_char_vec_hashes - .iter() - .map(|vec| vec.iter().take(i).collect::<String>()) - .collect(); - - let mut uniqnes_hashmap: HashSet<String> = HashSet::new(); - for ch in i_chars { - if !uniqnes_hashmap.insert(ch) { - // The key was already in the hash map, thus we have a duplicated char and need - // at least one char more - continue 'outer; - } - } - - needed_chars = i; - break 'outer; - } - - assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); - - Ok(needed_chars) - } -} diff --git a/crates/yt/src/storage/video_database/get/mod.rs b/crates/yt/src/storage/video_database/get/mod.rs deleted file mode 100644 index 0456cd3..0000000 --- a/crates/yt/src/storage/video_database/get/mod.rs +++ /dev/null @@ -1,307 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -//! These functions interact with the storage db in a read-only way. They are added on-demand (as -//! you could theoretically just could do everything with the `get_videos` function), as -//! performance or convince requires. -use std::{fs::File, path::PathBuf}; - -use anyhow::{Context, Result, bail}; -use blake3::Hash; -use log::{debug, trace}; -use sqlx::query; -use yt_dlp::InfoJson; - -use crate::{ - app::App, - storage::{ - subscriptions::Subscription, - video_database::{Video, extractor_hash::ExtractorHash}, - }, - unreachable::Unreachable, -}; - -use super::{MpvOptions, VideoOptions, VideoStatus, VideoStatusMarker, YtDlpOptions}; - -mod playlist; -pub use playlist::*; - -#[macro_export] -macro_rules! video_from_record { - ($record:expr) => { - Video { - description: $record.description.clone(), - duration: $crate::storage::video_database::MaybeDuration::from_maybe_secs_f64( - $record.duration, - ), - extractor_hash: - $crate::storage::video_database::extractor_hash::ExtractorHash::from_hash( - $record - .extractor_hash - .parse() - .expect("The db hash should be a valid blake3 hash"), - ), - last_status_change: $crate::storage::video_database::TimeStamp::from_secs( - $record.last_status_change, - ), - parent_subscription_name: $record.parent_subscription_name.clone(), - publish_date: $record - .publish_date - .map(|pd| $crate::storage::video_database::TimeStamp::from_secs(pd)), - status: { - let marker = $crate::storage::video_database::VideoStatusMarker::from_db_integer( - $record.status, - ); - - let optional = if let Some(cache_path) = &$record.cache_path { - Some(( - PathBuf::from(cache_path), - if $record.is_focused == Some(1) { - true - } else { - false - }, - )) - } else { - None - }; - - $crate::storage::video_database::VideoStatus::from_marker(marker, optional) - }, - thumbnail_url: if let Some(url) = &$record.thumbnail_url { - Some(url::Url::parse(&url).expect("Parsing this as url should always work")) - } else { - None - }, - title: $record.title.clone(), - url: url::Url::parse(&$record.url).expect("Parsing this as url should always work"), - priority: $crate::storage::video_database::Priority::from($record.priority), - - watch_progress: std::time::Duration::from_secs( - u64::try_from($record.watch_progress).expect("The record is positive i64"), - ), - } - }; -} - -/// Returns the videos that are in the `allowed_states`. -/// -/// # Panics -/// Only, if assertions fail. -pub async fn videos(app: &App, allowed_states: &[VideoStatusMarker]) -> Result<Vec<Video>> { - fn test(all_states: &[VideoStatusMarker], check: VideoStatusMarker) -> Option<i64> { - if all_states.contains(&check) { - trace!("State '{check:?}' marked as active"); - Some(check.as_db_integer()) - } else { - trace!("State '{check:?}' marked as inactive"); - None - } - } - fn states_to_string(allowed_states: &[VideoStatusMarker]) -> String { - let mut states = allowed_states - .iter() - .fold(String::from("&["), |mut acc, state| { - acc.push_str(state.as_str()); - acc.push_str(", "); - acc - }); - states = states.trim().to_owned(); - states = states.trim_end_matches(',').to_owned(); - states.push(']'); - states - } - - debug!( - "Fetching videos in the states: '{}'", - states_to_string(allowed_states) - ); - let active_pick: Option<i64> = test(allowed_states, VideoStatusMarker::Pick); - let active_watch: Option<i64> = test(allowed_states, VideoStatusMarker::Watch); - let active_cached: Option<i64> = test(allowed_states, VideoStatusMarker::Cached); - let active_watched: Option<i64> = test(allowed_states, VideoStatusMarker::Watched); - let active_drop: Option<i64> = test(allowed_states, VideoStatusMarker::Drop); - let active_dropped: Option<i64> = test(allowed_states, VideoStatusMarker::Dropped); - - let videos = query!( - r" - SELECT * - FROM videos - WHERE status IN (?,?,?,?,?,?) - ORDER BY priority DESC, publish_date DESC; - ", - active_pick, - active_watch, - active_cached, - active_watched, - active_drop, - active_dropped, - ) - .fetch_all(&app.database) - .await - .with_context(|| { - format!( - "Failed to query videos with states: '{}'", - states_to_string(allowed_states) - ) - })?; - - let real_videos: Vec<Video> = videos - .iter() - .map(|base| -> Video { - video_from_record! {base} - }) - .collect(); - - Ok(real_videos) -} - -pub fn video_info_json(video: &Video) -> Result<Option<InfoJson>> { - if let VideoStatus::Cached { mut cache_path, .. } = video.status.clone() { - if !cache_path.set_extension("info.json") { - bail!( - "Failed to change path extension to 'info.json': {}", - cache_path.display() - ); - } - let info_json_string = File::open(cache_path)?; - let info_json: InfoJson = serde_json::from_reader(&info_json_string)?; - - Ok(Some(info_json)) - } else { - Ok(None) - } -} - -pub async fn video_by_hash(app: &App, hash: &ExtractorHash) -> Result<Video> { - let ehash = hash.hash().to_string(); - - let raw_video = query!( - " - SELECT * FROM videos WHERE extractor_hash = ?; - ", - ehash - ) - .fetch_one(&app.database) - .await?; - - Ok(video_from_record! {raw_video}) -} - -pub async fn get_all_hashes(app: &App) -> Result<Vec<Hash>> { - let hashes_hex = query!( - r#" - SELECT extractor_hash - FROM videos; - "# - ) - .fetch_all(&app.database) - .await?; - - Ok(hashes_hex - .iter() - .map(|hash| { - Hash::from_hex(&hash.extractor_hash).unreachable( - "These values started as blake3 hashes, they should stay blake3 hashes", - ) - }) - .collect()) -} - -pub async fn get_video_hashes(app: &App, subs: &Subscription) -> Result<Vec<Hash>> { - let hashes_hex = query!( - r#" - SELECT extractor_hash - FROM videos - WHERE parent_subscription_name = ?; - "#, - subs.name - ) - .fetch_all(&app.database) - .await?; - - Ok(hashes_hex - .iter() - .map(|hash| { - Hash::from_hex(&hash.extractor_hash).unreachable( - "These values started as blake3 hashes, they should stay blake3 hashes", - ) - }) - .collect()) -} - -pub async fn get_video_yt_dlp_opts(app: &App, hash: &ExtractorHash) -> Result<YtDlpOptions> { - let ehash = hash.hash().to_string(); - - let yt_dlp_options = query!( - r#" - SELECT subtitle_langs - FROM video_options - WHERE extractor_hash = ?; - "#, - ehash - ) - .fetch_one(&app.database) - .await - .with_context(|| { - format!("Failed to fetch the `yt_dlp_video_opts` for video with hash: '{hash}'",) - })?; - - Ok(YtDlpOptions { - subtitle_langs: yt_dlp_options.subtitle_langs, - }) -} -pub async fn video_mpv_opts(app: &App, hash: &ExtractorHash) -> Result<MpvOptions> { - let ehash = hash.hash().to_string(); - - let mpv_options = query!( - r#" - SELECT playback_speed - FROM video_options - WHERE extractor_hash = ?; - "#, - ehash - ) - .fetch_one(&app.database) - .await - .with_context(|| { - format!("Failed to fetch the `mpv_video_opts` for video with hash: '{hash}'") - })?; - - Ok(MpvOptions { - playback_speed: mpv_options.playback_speed, - }) -} - -pub async fn get_video_opts(app: &App, hash: &ExtractorHash) -> Result<VideoOptions> { - let ehash = hash.hash().to_string(); - - let opts = query!( - r#" - SELECT playback_speed, subtitle_langs - FROM video_options - WHERE extractor_hash = ?; - "#, - ehash - ) - .fetch_one(&app.database) - .await - .with_context(|| format!("Failed to fetch the `video_opts` for video with hash: '{hash}'"))?; - - let mpv = MpvOptions { - playback_speed: opts.playback_speed, - }; - let yt_dlp = YtDlpOptions { - subtitle_langs: opts.subtitle_langs, - }; - - Ok(VideoOptions { yt_dlp, mpv }) -} diff --git a/crates/yt/src/storage/video_database/get/playlist/iterator.rs b/crates/yt/src/storage/video_database/get/playlist/iterator.rs deleted file mode 100644 index 4c45bf7..0000000 --- a/crates/yt/src/storage/video_database/get/playlist/iterator.rs +++ /dev/null @@ -1,101 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{ - collections::VecDeque, - path::{Path, PathBuf}, -}; - -use crate::storage::video_database::{Video, VideoStatus}; - -use super::Playlist; - -/// Turn a cached video into it's `cache_path` -fn to_cache_video(video: Video) -> PathBuf { - if let VideoStatus::Cached { cache_path, .. } = video.status { - cache_path - } else { - unreachable!("ALl of these videos should be cached.") - } -} - -#[derive(Debug)] -pub struct PlaylistIterator { - paths: VecDeque<PathBuf>, -} - -impl Iterator for PlaylistIterator { - type Item = <Playlist as IntoIterator>::Item; - - fn next(&mut self) -> Option<Self::Item> { - self.paths.pop_front() - } -} - -impl DoubleEndedIterator for PlaylistIterator { - fn next_back(&mut self) -> Option<Self::Item> { - self.paths.pop_back() - } -} - -impl IntoIterator for Playlist { - type Item = PathBuf; - - type IntoIter = PlaylistIterator; - - fn into_iter(self) -> Self::IntoIter { - let paths = self.videos.into_iter().map(to_cache_video).collect(); - Self::IntoIter { paths } - } -} - -#[derive(Debug)] -pub struct PlaylistIteratorBorrowed<'a> { - paths: Vec<&'a Path>, - index: usize, -} - -impl<'a> Iterator for PlaylistIteratorBorrowed<'a> { - type Item = <&'a Playlist as IntoIterator>::Item; - - fn next(&mut self) -> Option<Self::Item> { - let output = self.paths.get(self.index); - self.index += 1; - output.map(|v| &**v) - } -} - -impl<'a> Playlist { - #[must_use] - pub fn iter(&'a self) -> PlaylistIteratorBorrowed<'a> { - <&Self as IntoIterator>::into_iter(self) - } -} - -impl<'a> IntoIterator for &'a Playlist { - type Item = &'a Path; - - type IntoIter = PlaylistIteratorBorrowed<'a>; - - fn into_iter(self) -> Self::IntoIter { - let paths = self - .videos - .iter() - .map(|vid| { - if let VideoStatus::Cached { cache_path, .. } = &vid.status { - cache_path.as_path() - } else { - unreachable!("ALl of these videos should be cached.") - } - }) - .collect(); - Self::IntoIter { paths, index: 0 } - } -} diff --git a/crates/yt/src/storage/video_database/get/playlist/mod.rs b/crates/yt/src/storage/video_database/get/playlist/mod.rs deleted file mode 100644 index f6aadbf..0000000 --- a/crates/yt/src/storage/video_database/get/playlist/mod.rs +++ /dev/null @@ -1,167 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -//! This file contains the getters for the internal playlist - -use std::{ops::Add, path::PathBuf}; - -use crate::{ - app::App, - storage::video_database::{Video, VideoStatusMarker, extractor_hash::ExtractorHash}, - video_from_record, -}; - -use anyhow::Result; -use sqlx::query; - -pub mod iterator; - -/// Zero-based index into the internal playlist. -#[derive(Debug, Clone, Copy)] -pub struct PlaylistIndex(usize); - -impl From<PlaylistIndex> for usize { - fn from(value: PlaylistIndex) -> Self { - value.0 - } -} - -impl From<usize> for PlaylistIndex { - fn from(value: usize) -> Self { - Self(value) - } -} - -impl Add<usize> for PlaylistIndex { - type Output = Self; - - fn add(self, rhs: usize) -> Self::Output { - Self(self.0 + rhs) - } -} - -impl Add for PlaylistIndex { - type Output = Self; - - fn add(self, rhs: Self) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -/// A representation of the internal Playlist -#[derive(Debug)] -pub struct Playlist { - videos: Vec<Video>, -} - -impl Playlist { - /// Return the videos of this playlist. - #[must_use] - pub fn as_videos(&self) -> &[Video] { - &self.videos - } - - /// Turn this playlist to it's videos - #[must_use] - pub fn to_videos(self) -> Vec<Video> { - self.videos - } - - /// Find the index of the video specified by the `video_hash`. - /// - /// # Panics - /// Only if internal assertions fail. - #[must_use] - pub fn find_index(&self, video_hash: &ExtractorHash) -> Option<PlaylistIndex> { - if let Some((index, value)) = self - .videos - .iter() - .enumerate() - .find(|(_, other)| other.extractor_hash == *video_hash) - { - assert_eq!(value.extractor_hash, *video_hash); - Some(PlaylistIndex(index)) - } else { - None - } - } - - /// Select a video based on it's index - #[must_use] - pub fn get(&self, index: PlaylistIndex) -> Option<&Video> { - self.videos.get(index.0) - } - - /// Returns the number of videos in the playlist - #[must_use] - pub fn len(&self) -> usize { - self.videos.len() - } - /// Is the playlist empty? - #[must_use] - pub fn is_empty(&self) -> bool { - self.videos.is_empty() - } -} - -/// Return the current playlist index. -/// -/// This effectively looks for the currently focused video and returns it's index. -/// -/// # Panics -/// Only if internal assertions fail. -pub async fn current_playlist_index(app: &App) -> Result<Option<PlaylistIndex>> { - if let Some(focused) = currently_focused_video(app).await? { - let playlist = playlist(app).await?; - let index = playlist - .find_index(&focused.extractor_hash) - .expect("All focused videos must also be in the playlist"); - Ok(Some(index)) - } else { - Ok(None) - } -} - -/// Return the video in the playlist at the position `index`. -pub async fn playlist_entry(app: &App, index: PlaylistIndex) -> Result<Option<Video>> { - let playlist = playlist(app).await?; - - if let Some(vid) = playlist.get(index) { - Ok(Some(vid.to_owned())) - } else { - Ok(None) - } -} - -pub async fn playlist(app: &App) -> Result<Playlist> { - let videos = super::videos(app, &[VideoStatusMarker::Cached]).await?; - - Ok(Playlist { videos }) -} - -/// This returns the video with the `is_focused` flag set. -/// # Panics -/// Only if assertions fail. -pub async fn currently_focused_video(app: &App) -> Result<Option<Video>> { - let cached_status = VideoStatusMarker::Cached.as_db_integer(); - let record = query!( - "SELECT * FROM videos WHERE is_focused = 1 AND status = ?", - cached_status - ) - .fetch_one(&app.database) - .await; - - if let Err(sqlx::Error::RowNotFound) = record { - Ok(None) - } else { - let base = record?; - Ok(Some(video_from_record! {base})) - } -} diff --git a/crates/yt/src/storage/video_database/set/mod.rs b/crates/yt/src/storage/video_database/set/mod.rs deleted file mode 100644 index 8c1be4a..0000000 --- a/crates/yt/src/storage/video_database/set/mod.rs +++ /dev/null @@ -1,333 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -//! These functions change the database. They are added on a demand basis. - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use chrono::Utc; -use log::{debug, info}; -use sqlx::query; -use tokio::fs; - -use crate::{app::App, storage::video_database::extractor_hash::ExtractorHash, video_from_record}; - -use super::{Priority, Video, VideoOptions, VideoStatus}; - -mod playlist; -pub use playlist::*; - -const fn is_focused_to_value(is_focused: bool) -> Option<i8> { - if is_focused { Some(1) } else { None } -} - -/// Set a new status for a video. -/// This will only update the status time stamp/priority when the status or the priority has changed . -pub async fn video_status( - app: &App, - video_hash: &ExtractorHash, - new_status: VideoStatus, - new_priority: Option<Priority>, -) -> Result<()> { - let video_hash = video_hash.hash().to_string(); - - let old = { - let base = query!( - r#" - SELECT * - FROM videos - WHERE extractor_hash = ? - "#, - video_hash - ) - .fetch_one(&app.database) - .await?; - - video_from_record! {base} - }; - - let old_marker = old.status.as_marker(); - let (cache_path, is_focused) = { - fn cache_path_to_string(path: &Path) -> Result<String> { - Ok(path - .to_str() - .with_context(|| { - format!( - "Failed to parse cache path ('{}') as utf8 string", - path.display() - ) - })? - .to_owned()) - } - - if let VideoStatus::Cached { - cache_path, - is_focused, - } = &new_status - { - ( - Some(cache_path_to_string(cache_path)?), - is_focused_to_value(*is_focused), - ) - } else { - (None, None) - } - }; - - let new_status = new_status.as_marker(); - - if let Some(new_priority) = new_priority { - if old_marker == new_status && old.priority == new_priority { - return Ok(()); - } - - let now = Utc::now().timestamp(); - - debug!( - "Running status change: {:#?} -> {:#?}...", - old_marker, new_status, - ); - - let new_status = new_status.as_db_integer(); - let new_priority = new_priority.as_db_integer(); - query!( - r#" - UPDATE videos - SET status = ?, last_status_change = ?, priority = ?, cache_path = ?, is_focused = ? - WHERE extractor_hash = ?; - "#, - new_status, - now, - new_priority, - cache_path, - is_focused, - video_hash - ) - .execute(&app.database) - .await?; - } else { - if old_marker == new_status { - return Ok(()); - } - - let now = Utc::now().timestamp(); - - debug!( - "Running status change: {:#?} -> {:#?}...", - old_marker, new_status, - ); - - let new_status = new_status.as_db_integer(); - query!( - r#" - UPDATE videos - SET status = ?, last_status_change = ?, cache_path = ?, is_focused = ? - WHERE extractor_hash = ?; - "#, - new_status, - now, - cache_path, - is_focused, - video_hash - ) - .execute(&app.database) - .await?; - } - - debug!("Finished status change."); - Ok(()) -} - -/// Mark a video as watched. -/// This will both set the status to `Watched` and the `cache_path` to Null. -/// -/// # Panics -/// Only if assertions fail. -pub async fn video_watched(app: &App, video: &ExtractorHash) -> Result<()> { - let old = { - let video_hash = video.hash().to_string(); - - let base = query!( - r#" - SELECT * - FROM videos - WHERE extractor_hash = ? - "#, - video_hash - ) - .fetch_one(&app.database) - .await?; - - video_from_record! {base} - }; - - info!("Will set video watched: '{}'", old.title); - - if let VideoStatus::Cached { cache_path, .. } = &old.status { - if let Ok(true) = cache_path.try_exists() { - fs::remove_file(cache_path).await?; - } - } else { - unreachable!("The video must be marked as Cached before it can be marked Watched"); - } - - video_status(app, video, VideoStatus::Watched, None).await?; - - Ok(()) -} - -pub(crate) async fn video_watch_progress( - app: &App, - extractor_hash: &ExtractorHash, - watch_progress: u32, -) -> std::result::Result<(), anyhow::Error> { - let video_extractor_hash = extractor_hash.hash().to_string(); - - query!( - r#" - UPDATE videos - SET watch_progress = ? - WHERE extractor_hash = ?; - "#, - watch_progress, - video_extractor_hash, - ) - .execute(&app.database) - .await?; - - Ok(()) -} - -pub async fn set_video_options( - app: &App, - hash: &ExtractorHash, - video_options: &VideoOptions, -) -> Result<()> { - let video_extractor_hash = hash.hash().to_string(); - let playback_speed = video_options.mpv.playback_speed; - let subtitle_langs = &video_options.yt_dlp.subtitle_langs; - - query!( - r#" - UPDATE video_options - SET playback_speed = ?, subtitle_langs = ? - WHERE extractor_hash = ?; - "#, - playback_speed, - subtitle_langs, - video_extractor_hash, - ) - .execute(&app.database) - .await?; - - Ok(()) -} - -/// # Panics -/// Only if internal expectations fail. -pub async fn add_video(app: &App, video: Video) -> Result<()> { - let parent_subscription_name = video.parent_subscription_name; - - let thumbnail_url = video.thumbnail_url.map(|val| val.to_string()); - - let url = video.url.to_string(); - let extractor_hash = video.extractor_hash.hash().to_string(); - - let default_subtitle_langs = &app.config.select.subtitle_langs; - let default_mpv_playback_speed = app.config.select.playback_speed; - - let status = video.status.as_marker().as_db_integer(); - let (cache_path, is_focused) = if let VideoStatus::Cached { - cache_path, - is_focused, - } = video.status - { - ( - Some( - cache_path - .to_str() - .with_context(|| { - format!( - "Failed to prase cache path '{}' as utf-8 string", - cache_path.display() - ) - })? - .to_string(), - ), - is_focused_to_value(is_focused), - ) - } else { - (None, None) - }; - - let duration: Option<f64> = video.duration.as_secs_f64(); - let last_status_change: i64 = video.last_status_change.as_secs(); - let publish_date: Option<i64> = video.publish_date.map(|pd| pd.as_secs()); - let watch_progress: i64 = - i64::try_from(video.watch_progress.as_secs()).expect("This should never exceed a u32"); - - let mut tx = app.database.begin().await?; - query!( - r#" - INSERT INTO videos ( - description, - duration, - extractor_hash, - is_focused, - last_status_change, - parent_subscription_name, - publish_date, - status, - thumbnail_url, - title, - url, - watch_progress, - cache_path - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); - "#, - video.description, - duration, - extractor_hash, - is_focused, - last_status_change, - parent_subscription_name, - publish_date, - status, - thumbnail_url, - video.title, - url, - watch_progress, - cache_path, - ) - .execute(&mut *tx) - .await?; - - query!( - r#" - INSERT INTO video_options ( - extractor_hash, - subtitle_langs, - playback_speed) - VALUES (?, ?, ?); - "#, - extractor_hash, - default_subtitle_langs, - default_mpv_playback_speed - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(()) -} diff --git a/crates/yt/src/storage/video_database/set/playlist.rs b/crates/yt/src/storage/video_database/set/playlist.rs deleted file mode 100644 index 547df21..0000000 --- a/crates/yt/src/storage/video_database/set/playlist.rs +++ /dev/null @@ -1,101 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use anyhow::Result; -use log::debug; -use sqlx::query; - -use crate::{ - app::App, - storage::video_database::{extractor_hash::ExtractorHash, get}, -}; - -/// Set a video to be focused. -/// This optionally takes another video hash, which marks the old focused video. -/// This will then be disabled. -/// -/// # Panics -/// Only if internal assertions fail. -pub async fn focused( - app: &App, - new_video_hash: &ExtractorHash, - old_video_hash: Option<&ExtractorHash>, -) -> Result<()> { - unfocused(app, old_video_hash).await?; - - debug!("Focusing video: '{new_video_hash}'"); - let new_hash = new_video_hash.hash().to_string(); - query!( - r#" - UPDATE videos - SET is_focused = 1 - WHERE extractor_hash = ?; - "#, - new_hash, - ) - .execute(&app.database) - .await?; - - assert_eq!( - *new_video_hash, - get::currently_focused_video(app) - .await? - .expect("This is some at this point") - .extractor_hash - ); - Ok(()) -} - -/// Set a video to be no longer focused. -/// This will use the supplied `video_hash` if it is [`Some`], otherwise it will simply un-focus -/// the currently focused video. -/// -/// # Panics -/// Only if internal assertions fail. -pub async fn unfocused(app: &App, video_hash: Option<&ExtractorHash>) -> Result<()> { - let hash = if let Some(hash) = video_hash { - hash.hash().to_string() - } else { - let output = query!( - r#" - SELECT extractor_hash - FROM videos - WHERE is_focused = 1; - "#, - ) - .fetch_optional(&app.database) - .await?; - - if let Some(output) = output { - output.extractor_hash - } else { - // There is no unfocused video right now. - return Ok(()); - } - }; - debug!("Unfocusing video: '{hash}'"); - - query!( - r#" - UPDATE videos - SET is_focused = NULL - WHERE extractor_hash = ?; - "#, - hash - ) - .execute(&app.database) - .await?; - - assert!( - get::currently_focused_video(app).await?.is_none(), - "We assumed that the video we just removed was actually a focused one." - ); - Ok(()) -} diff --git a/crates/yt/src/subscribe/mod.rs b/crates/yt/src/subscribe/mod.rs deleted file mode 100644 index 7ac0be4..0000000 --- a/crates/yt/src/subscribe/mod.rs +++ /dev/null @@ -1,184 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::str::FromStr; - -use anyhow::{Context, Result, bail}; -use futures::FutureExt; -use log::warn; -use tokio::io::{AsyncBufRead, AsyncBufReadExt}; -use url::Url; -use yt_dlp::{YoutubeDLOptions, json_get}; - -use crate::{ - app::App, - storage::subscriptions::{ - Subscription, add_subscription, check_url, get, remove_all, remove_subscription, - }, - unreachable::Unreachable, -}; - -pub async fn unsubscribe(app: &App, name: String) -> Result<()> { - let present_subscriptions = get(app).await?; - - if let Some(subscription) = present_subscriptions.0.get(&name) { - remove_subscription(app, subscription).await?; - } else { - bail!("Couldn't find subscription: '{}'", &name); - } - - Ok(()) -} - -pub async fn import<W: AsyncBufRead + AsyncBufReadExt + Unpin>( - app: &App, - reader: W, - force: bool, -) -> Result<()> { - if force { - remove_all(app).await?; - } - - let mut lines = reader.lines(); - while let Some(line) = lines.next_line().await? { - let url = - Url::from_str(&line).with_context(|| format!("Failed to parse '{line}' as url"))?; - match subscribe(app, None, url) - .await - .with_context(|| format!("Failed to subscribe to: '{line}'")) - { - Ok(()) => (), - Err(err) => eprintln!( - "Error while subscribing to '{}': '{}'", - line, - err.source().unreachable("Should have a source") - ), - } - } - - Ok(()) -} - -pub async fn subscribe(app: &App, name: Option<String>, url: Url) -> Result<()> { - if !(url.as_str().ends_with("videos") - || url.as_str().ends_with("streams") - || url.as_str().ends_with("shorts") - || url.as_str().ends_with("videos/") - || url.as_str().ends_with("streams/") - || url.as_str().ends_with("shorts/")) - && url.as_str().contains("youtube.com") - { - warn!( - "Your youtbe url does not seem like it actually tracks a channels playlist (videos, streams, shorts). Adding subscriptions for each of them..." - ); - - let url = Url::parse(&(url.as_str().to_owned() + "/")) - .unreachable("This was an url, it should stay one"); - - if let Some(name) = name { - let out: Result<()> = async move { - actual_subscribe( - app, - Some(name.clone() + " {Videos}"), - url.join("videos/") - .unreachable("The url should allow being joined onto"), - ) - .await - .with_context(|| { - format!("Failed to subscribe to '{}'", name.clone() + " {Videos}") - })?; - - actual_subscribe( - app, - Some(name.clone() + " {Streams}"), - url.join("streams/").unreachable("See above."), - ) - .await - .with_context(|| { - format!("Failed to subscribe to '{}'", name.clone() + " {Streams}") - })?; - - actual_subscribe( - app, - Some(name.clone() + " {Shorts}"), - url.join("shorts/").unreachable("See above."), - ) - .await - .with_context(|| format!("Failed to subscribe to '{}'", name + " {Shorts}"))?; - - Ok(()) - } - .boxed() - .await; - - out?; - } else { - actual_subscribe(app, None, url.join("videos/").unreachable("See above.")) - .await - .with_context(|| format!("Failed to subscribe to the '{}' variant", "{Videos}"))?; - - actual_subscribe(app, None, url.join("streams/").unreachable("See above.")) - .await - .with_context(|| format!("Failed to subscribe to the '{}' variant", "{Streams}"))?; - - actual_subscribe(app, None, url.join("shorts/").unreachable("See above.")) - .await - .with_context(|| format!("Failed to subscribe to the '{}' variant", "{Shorts}"))?; - } - } else { - actual_subscribe(app, name, url).await?; - } - - Ok(()) -} - -async fn actual_subscribe(app: &App, name: Option<String>, url: Url) -> Result<()> { - if !check_url(url.clone()).await? { - bail!("The url ('{}') does not represent a playlist!", &url) - } - - let name = if let Some(name) = name { - name - } else { - let yt_dlp = YoutubeDLOptions::new() - .set("playliststart", 1) - .set("playlistend", 10) - .set("noplaylist", false) - .set("extract_flat", "in_playlist") - .build()?; - - let info = yt_dlp.extract_info(&url, false, false)?; - - if info.get("_type") == Some(&serde_json::Value::String("Playlist".to_owned())) { - json_get!(info, "title", as_str).to_owned() - } else { - bail!("The url ('{}') does not represent a playlist!", &url) - } - }; - - let present_subscriptions = get(app).await?; - - if let Some(subs) = present_subscriptions.0.get(&name) { - bail!( - "The subscription '{}' could not be added, \ - as another one with the same name ('{}') already exists. It links to the Url: '{}'", - name, - name, - subs.url - ); - } - - let sub = Subscription { name, url }; - - add_subscription(app, &sub).await?; - - Ok(()) -} diff --git a/crates/yt/src/unreachable.rs b/crates/yt/src/unreachable.rs deleted file mode 100644 index 436fbb6..0000000 --- a/crates/yt/src/unreachable.rs +++ /dev/null @@ -1,50 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// This has been taken from: https://gitlab.torproject.org/tpo/core/arti/-/issues/950 - -// The functions here should be annotated with `#[inline(always)]`. -#![allow(clippy::inline_always)] - -use std::fmt::Debug; - -/// Trait for something that can possibly be unwrapped, like a Result or Option. -/// It provides semantic information, that unwrapping here should never happen. -pub trait Unreachable<T> { - /// Like `expect()`, but does not trigger clippy. - /// - /// # Usage - /// - /// This method only exists so that we can use it without hitting - /// `clippy::missing_panics_docs`. Therefore, we should only use it - /// for situations where we are certain that the panic cannot occur - /// unless something else is very broken. Consider instead using - /// `expect()` and adding a `Panics` section to your function - /// documentation. - /// - /// # Panics - /// - /// Panics if this is not an object that can be unwrapped, such as - /// None or an Err. - fn unreachable(self, msg: &str) -> T; -} -impl<T> Unreachable<T> for Option<T> { - #[inline(always)] - fn unreachable(self, msg: &str) -> T { - self.expect(msg) - } -} -impl<T, E: Debug> Unreachable<T> for Result<T, E> { - #[inline(always)] - fn unreachable(self, msg: &str) -> T { - self.expect(msg) - } -} diff --git a/crates/yt/src/update/mod.rs b/crates/yt/src/update/mod.rs deleted file mode 100644 index d866882..0000000 --- a/crates/yt/src/update/mod.rs +++ /dev/null @@ -1,204 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{str::FromStr, time::Duration}; - -use anyhow::{Context, Ok, Result}; -use chrono::{DateTime, Utc}; -use log::warn; -use url::Url; -use yt_dlp::{InfoJson, json_cast, json_get}; - -use crate::{ - app::App, - select::selection_file::duration::MaybeDuration, - storage::{ - subscriptions::{self, Subscription}, - video_database::{ - Priority, TimeStamp, Video, VideoStatus, extractor_hash::ExtractorHash, - get::get_all_hashes, set::add_video, - }, - }, -}; - -mod updater; -use updater::Updater; - -pub async fn update( - app: &App, - max_backlog: usize, - subscription_names_to_update: Vec<String>, - total_number: Option<usize>, - current_progress: Option<usize>, -) -> Result<()> { - let subscriptions = subscriptions::get(app).await?; - - let subs: Vec<Subscription> = if subscription_names_to_update.is_empty() { - subscriptions.0.into_values().collect() - } else { - subscriptions - .0 - .into_values() - .filter(|sub| subscription_names_to_update.contains(&sub.name)) - .collect() - }; - - // We can get away with not having to re-fetch the hashes every time, as the returned video - // should not contain duplicates. - let hashes = get_all_hashes(app).await?; - - let updater = Updater::new(max_backlog, hashes); - updater - .update(app, subs, total_number, current_progress) - .await?; - - Ok(()) -} - -#[allow(clippy::too_many_lines)] -pub fn video_entry_to_video(entry: &InfoJson, sub: Option<&Subscription>) -> Result<Video> { - fn fmt_context(date: &str, extended: Option<&str>) -> String { - let f = format!( - "Failed to parse the `upload_date` of the entry ('{date}'). \ - Expected `YYYY-MM-DD`, has the format changed?" - ); - if let Some(date_string) = extended { - format!("{f}\nThe parsed '{date_string}' can't be turned to a valid UTC date.'") - } else { - f - } - } - - let publish_date = if let Some(date) = &entry.get("upload_date") { - let date = json_cast!(date, as_str); - - let year: u32 = date - .chars() - .take(4) - .collect::<String>() - .parse() - .with_context(|| fmt_context(date, None))?; - let month: u32 = date - .chars() - .skip(4) - .take(2) - .collect::<String>() - .parse() - .with_context(|| fmt_context(date, None))?; - let day: u32 = date - .chars() - .skip(4 + 2) - .take(2) - .collect::<String>() - .parse() - .with_context(|| fmt_context(date, None))?; - - let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z"); - Some( - DateTime::<Utc>::from_str(&date_string) - .with_context(|| fmt_context(date, Some(&date_string)))? - .timestamp(), - ) - } else { - warn!( - "The video '{}' lacks it's upload date!", - json_get!(entry, "title", as_str) - ); - None - }; - - let thumbnail_url = match (&entry.get("thumbnails"), &entry.get("thumbnail")) { - (None, None) => None, - (None, Some(thumbnail)) => Some(Url::from_str(json_cast!(thumbnail, as_str))?), - - // TODO: The algorithm is not exactly the best <2024-05-28> - (Some(thumbnails), None) => { - if let Some(thumbnail) = json_cast!(thumbnails, as_array).first() { - Some(Url::from_str(json_get!( - json_cast!(thumbnail, as_object), - "url", - as_str - ))?) - } else { - None - } - } - (Some(_), Some(thumnail)) => Some(Url::from_str(json_cast!(thumnail, as_str))?), - }; - - let url = { - let smug_url: Url = json_get!(entry, "webpage_url", as_str).parse()?; - // TODO(@bpeetz): We should probably add this? <2025-06-14> - // if '#__youtubedl_smuggle' not in smug_url: - // return smug_url, default - // url, _, sdata = smug_url.rpartition('#') - // jsond = urllib.parse.parse_qs(sdata)['__youtubedl_smuggle'][0] - // data = json.loads(jsond) - // return url, data - - smug_url - }; - - let extractor_hash = blake3::hash(json_get!(entry, "id", as_str).as_bytes()); - - let subscription_name = if let Some(sub) = sub { - Some(sub.name.clone()) - } else if let Some(uploader) = entry.get("uploader").map(|val| json_cast!(val, as_str)) { - if entry - .get("webpage_url_domain") - .map(|val| json_cast!(val, as_str)) - == Some("youtube.com") - { - Some(format!("{uploader} - Videos")) - } else { - Some(uploader.to_owned()) - } - } else { - None - }; - - let video = Video { - description: entry - .get("description") - .map(|val| json_cast!(val, as_str).to_owned()), - duration: MaybeDuration::from_maybe_secs_f64( - entry.get("duration").map(|val| json_cast!(val, as_f64)), - ), - extractor_hash: ExtractorHash::from_hash(extractor_hash), - last_status_change: TimeStamp::from_now(), - parent_subscription_name: subscription_name, - priority: Priority::default(), - publish_date: publish_date.map(TimeStamp::from_secs), - status: VideoStatus::Pick, - thumbnail_url, - title: json_get!(entry, "title", as_str).to_owned(), - url, - watch_progress: Duration::default(), - }; - Ok(video) -} - -async fn process_subscription(app: &App, sub: Subscription, entry: InfoJson) -> Result<()> { - let video = video_entry_to_video(&entry, Some(&sub)) - .context("Failed to parse search entry as Video")?; - - add_video(app, video.clone()) - .await - .with_context(|| format!("Failed to add video to database: '{}'", video.title))?; - println!( - "{}", - &video - .to_line_display(app) - .await - .with_context(|| format!("Failed to format video: '{}'", video.title))? - ); - Ok(()) -} diff --git a/crates/yt/src/update/updater.rs b/crates/yt/src/update/updater.rs deleted file mode 100644 index 04bcaa1..0000000 --- a/crates/yt/src/update/updater.rs +++ /dev/null @@ -1,187 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{ - io::{Write, stderr}, - sync::atomic::{AtomicUsize, Ordering}, -}; - -use anyhow::{Context, Result}; -use blake3::Hash; -use futures::{StreamExt, future::join_all, stream}; -use log::{Level, debug, error, log_enabled}; -use serde_json::json; -use tokio_util::task::LocalPoolHandle; -use yt_dlp::{InfoJson, YoutubeDLOptions, json_cast, json_get, process_ie_result}; - -use crate::{ - ansi_escape_codes::{clear_whole_line, move_to_col}, - app::App, - storage::subscriptions::Subscription, -}; - -use super::process_subscription; - -pub(super) struct Updater { - max_backlog: usize, - hashes: Vec<Hash>, - pool: LocalPoolHandle, -} - -static REACHED_NUMBER: AtomicUsize = const { AtomicUsize::new(1) }; - -impl Updater { - pub(super) fn new(max_backlog: usize, hashes: Vec<Hash>) -> Self { - // TODO(@bpeetz): The number should not be hardcoded. <2025-06-14> - let pool = LocalPoolHandle::new(16); - - Self { - max_backlog, - hashes, - pool, - } - } - - pub(super) async fn update( - self, - app: &App, - subscriptions: Vec<Subscription>, - total_number: Option<usize>, - current_progress: Option<usize>, - ) -> Result<()> { - let total_number = total_number.unwrap_or(subscriptions.len()); - - if let Some(current_progress) = current_progress { - REACHED_NUMBER.store(current_progress, Ordering::Relaxed); - } - - let mut stream = stream::iter(subscriptions) - .map(|sub| self.get_new_entries(sub, total_number)) - .buffer_unordered(16 * 4); - - while let Some(output) = stream.next().await { - let mut entries = output?; - - if let Some(next) = entries.next() { - let (sub, entry) = next; - process_subscription(app, sub, entry).await?; - - join_all(entries.map(|(sub, entry)| process_subscription(app, sub, entry))) - .await - .into_iter() - .collect::<Result<(), _>>()?; - } - } - - Ok(()) - } - - async fn get_new_entries( - &self, - sub: Subscription, - total_number: usize, - ) -> Result<impl Iterator<Item = (Subscription, InfoJson)>> { - let max_backlog = self.max_backlog; - let hashes = self.hashes.clone(); - - let yt_dlp = YoutubeDLOptions::new() - .set("playliststart", 1) - .set("playlistend", max_backlog) - .set("noplaylist", false) - .set( - "extractor_args", - json! {{"youtubetab": {"approximate_date": [""]}}}, - ) - // TODO: This also removes unlisted and other stuff. Find a good way to remove the - // members-only videos from the feed. <2025-04-17> - .set("match-filter", "availability=public") - .build()?; - - self.pool - .spawn_pinned(move || { - async move { - if !log_enabled!(Level::Debug) { - clear_whole_line(); - move_to_col(1); - eprint!( - "({}/{total_number}) Checking playlist {}...", - REACHED_NUMBER.fetch_add(1, Ordering::Relaxed), - sub.name - ); - move_to_col(1); - stderr().flush()?; - } - - let info = yt_dlp - .extract_info(&sub.url, false, false) - .with_context(|| format!("Failed to get playlist '{}'.", sub.name))?; - - let empty = vec![]; - let entries = info - .get("entries") - .map_or(&empty, |val| json_cast!(val, as_array)); - - let valid_entries: Vec<(Subscription, InfoJson)> = entries - .iter() - .take(max_backlog) - .filter_map(|entry| -> Option<(Subscription, InfoJson)> { - let id = json_get!(entry, "id", as_str); - let extractor_hash = blake3::hash(id.as_bytes()); - - if hashes.contains(&extractor_hash) { - debug!( - "Skipping entry, as it is already present: '{extractor_hash}'", - ); - None - } else { - Some((sub.clone(), json_cast!(entry, as_object).to_owned())) - } - }) - .collect(); - - Ok(valid_entries - .into_iter() - .map(|(sub, entry)| { - let inner_yt_dlp = YoutubeDLOptions::new() - .set("noplaylist", true) - .build() - .expect("Worked before, should work now"); - - match inner_yt_dlp.process_ie_result(entry, false) { - Ok(output) => Ok((sub, output)), - Err(err) => Err(err), - } - }) - // Don't fail the whole update, if one of the entries fails to fetch. - .filter_map(|base| match base { - Ok(ok) => Some(ok), - Err(err) => { - let process_ie_result::Error::Python(err) = &err; - - if err.contains( - "Join this channel to get access to members-only content ", - ) { - // Hide this error - } else { - // Show the error, but don't fail. - let error = err - .strip_prefix("DownloadError: \u{1b}[0;31mERROR:\u{1b}[0m ") - .unwrap_or(err); - error!("{error}"); - } - - None - } - })) - } - }) - .await? - } -} diff --git a/crates/yt/src/version/mod.rs b/crates/yt/src/version/mod.rs index 9a91f3b..b12eadd 100644 --- a/crates/yt/src/version/mod.rs +++ b/crates/yt/src/version/mod.rs @@ -10,11 +10,11 @@ use anyhow::{Context, Result}; use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; -use yt_dlp::YoutubeDLOptions; +use yt_dlp::options::YoutubeDLOptions; use crate::{config::Config, storage::migrate::get_version_db}; -pub async fn show(config: &Config) -> Result<()> { +pub(crate) async fn show(config: &Config) -> Result<()> { let db_version = { let options = SqliteConnectOptions::new() .filename(&config.paths.database_path) @@ -30,17 +30,20 @@ pub async fn show(config: &Config) -> Result<()> { .context("Failed to determine database version")? }; - let yt_dlp_version = { + let (yt_dlp, python) = { let yt_dlp = YoutubeDLOptions::new().build()?; - yt_dlp.version() + yt_dlp.version()? }; + let python = python.replace('\n', " "); + println!( "{}: {} db version: {db_version} -yt-dlp: {yt_dlp_version}", +yt-dlp: {yt_dlp} +python: {python}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"), ); diff --git a/crates/yt/src/videos/display/format_video.rs b/crates/yt/src/videos/display/format_video.rs deleted file mode 100644 index b97acb1..0000000 --- a/crates/yt/src/videos/display/format_video.rs +++ /dev/null @@ -1,94 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use anyhow::Result; - -use crate::{app::App, comments::output::format_text, storage::video_database::Video}; - -impl Video { - pub async fn to_info_display(&self, app: &App) -> Result<String> { - let cache_path = self.cache_path_fmt(app); - let description = self.description_fmt(); - let duration = self.duration_fmt(app); - let extractor_hash = self.extractor_hash_fmt(app).await?; - let in_playlist = self.in_playlist_fmt(app); - let last_status_change = self.last_status_change_fmt(app); - let parent_subscription_name = self.parent_subscription_name_fmt(app); - let priority = self.priority_fmt(); - let publish_date = self.publish_date_fmt(app); - let status = self.status_fmt(app); - let thumbnail_url = self.thumbnail_url_fmt(); - let title = self.title_fmt(app); - let url = self.url_fmt(app); - let watch_progress = self.watch_progress_fmt(app); - let video_options = self.video_options_fmt(app).await?; - - let watched_percentage_fmt = { - if let Some(duration) = self.duration.as_secs() { - format!( - " (watched: {:0.0}%)", - (self.watch_progress.as_secs() / duration) * 100 - ) - } else { - format!(" {watch_progress}") - } - }; - - let string = format!( - "\ -{title} ({extractor_hash}) -| -> {cache_path} -| -> {duration}{watched_percentage_fmt} -| -> {parent_subscription_name} -| -> priority: {priority} -| -> {publish_date} -| -> status: {status} since {last_status_change} ({in_playlist}) -| -> {thumbnail_url} -| -> {url} -| -> options: {} -{}\n", - video_options.to_string().trim(), - format_text(description.to_string().as_str()) - ); - Ok(string) - } - - pub async fn to_line_display(&self, app: &App) -> Result<String> { - let f = format!( - "{} {} {} {} {} {}", - self.status_fmt(app), - self.extractor_hash_fmt(app).await?, - self.title_fmt(app), - self.publish_date_fmt(app), - self.parent_subscription_name_fmt(app), - self.duration_fmt(app) - ); - - Ok(f) - } - - pub async fn to_select_file_display(&self, app: &App) -> Result<String> { - let f = format!( - r#"{}{} {} "{}" "{}" "{}" "{}" "{}"{}"#, - self.status_fmt_no_color(), - self.video_options_fmt_no_color(app).await?, - self.extractor_hash_fmt_no_color(app).await?, - self.title_fmt_no_color(), - self.publish_date_fmt_no_color(), - self.parent_subscription_name_fmt_no_color(), - self.duration_fmt_no_color(), - self.url_fmt_no_color(), - '\n' - ); - - Ok(f) - } -} diff --git a/crates/yt/src/videos/display/mod.rs b/crates/yt/src/videos/display/mod.rs deleted file mode 100644 index 1188569..0000000 --- a/crates/yt/src/videos/display/mod.rs +++ /dev/null @@ -1,229 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use owo_colors::OwoColorize; -use url::Url; - -use crate::{ - app::App, - select::selection_file::duration::MaybeDuration, - storage::video_database::{TimeStamp, Video, VideoStatus, get::get_video_opts}, -}; - -use anyhow::{Context, Result}; - -pub mod format_video; - -macro_rules! get { - ($value:expr, $key:ident, $name:expr, $code:tt) => { - if let Some(value) = &$value.$key { - $code(value) - } else { - concat!("[No ", $name, "]").to_owned() - } - }; -} - -fn maybe_add_color<F>(app: &App, input: String, mut color_fn: F) -> String -where - F: FnMut(String) -> String, -{ - if app.config.global.display_colors { - color_fn(input) - } else { - input - } -} -impl Video { - #[must_use] - pub fn cache_path_fmt(&self, app: &App) -> String { - let cache_path = if let VideoStatus::Cached { - cache_path, - is_focused: _, - } = &self.status - { - cache_path.to_string_lossy().to_string() - } else { - "[No Cache Path]".to_owned() - }; - maybe_add_color(app, cache_path, |v| v.blue().bold().to_string()) - } - - #[must_use] - pub fn description_fmt(&self) -> String { - get!( - self, - description, - "Description", - (|value: &str| value.to_owned()) - ) - } - - #[must_use] - pub fn duration_fmt_no_color(&self) -> String { - self.duration.to_string() - } - #[must_use] - pub fn duration_fmt(&self, app: &App) -> String { - let duration = self.duration_fmt_no_color(); - maybe_add_color(app, duration, |v| v.cyan().bold().to_string()) - } - - #[must_use] - pub fn watch_progress_fmt(&self, app: &App) -> String { - maybe_add_color( - app, - MaybeDuration::from_std(self.watch_progress).to_string(), - |v| v.cyan().bold().to_string(), - ) - } - - pub async fn extractor_hash_fmt_no_color(&self, app: &App) -> Result<String> { - let hash = self - .extractor_hash - .into_short_hash(app) - .await - .with_context(|| { - format!( - "Failed to format extractor hash, whilst formatting video: '{}'", - self.title - ) - })? - .to_string(); - Ok(hash) - } - pub async fn extractor_hash_fmt(&self, app: &App) -> Result<String> { - let hash = self.extractor_hash_fmt_no_color(app).await?; - Ok(maybe_add_color(app, hash, |v| { - v.bright_purple().italic().to_string() - })) - } - - #[must_use] - pub fn in_playlist_fmt(&self, app: &App) -> String { - let output = match &self.status { - VideoStatus::Pick - | VideoStatus::Watch - | VideoStatus::Watched - | VideoStatus::Drop - | VideoStatus::Dropped => "Not in the playlist", - VideoStatus::Cached { is_focused, .. } => { - if *is_focused { - "In the playlist and focused" - } else { - "In the playlist" - } - } - }; - maybe_add_color(app, output.to_owned(), |v| v.yellow().italic().to_string()) - } - #[must_use] - pub fn last_status_change_fmt(&self, app: &App) -> String { - maybe_add_color(app, self.last_status_change.to_string(), |v| { - v.bright_cyan().to_string() - }) - } - - #[must_use] - pub fn parent_subscription_name_fmt_no_color(&self) -> String { - get!( - self, - parent_subscription_name, - "author", - (|sub: &str| sub.replace('"', "'")) - ) - } - #[must_use] - pub fn parent_subscription_name_fmt(&self, app: &App) -> String { - let psn = self.parent_subscription_name_fmt_no_color(); - maybe_add_color(app, psn, |v| v.bright_magenta().to_string()) - } - - #[must_use] - pub fn priority_fmt(&self) -> String { - self.priority.to_string() - } - - #[must_use] - pub fn publish_date_fmt_no_color(&self) -> String { - get!( - self, - publish_date, - "release date", - (|date: &TimeStamp| date.to_string()) - ) - } - #[must_use] - pub fn publish_date_fmt(&self, app: &App) -> String { - let date = self.publish_date_fmt_no_color(); - maybe_add_color(app, date, |v| v.bright_white().bold().to_string()) - } - - #[must_use] - pub fn status_fmt_no_color(&self) -> String { - // TODO: We might support `.trim()`ing that, as the extra whitespace could be bad in the - // selection file. <2024-10-07> - self.status.as_marker().as_command().to_string() - } - #[must_use] - pub fn status_fmt(&self, app: &App) -> String { - let status = self.status_fmt_no_color(); - maybe_add_color(app, status, |v| v.red().bold().to_string()) - } - - #[must_use] - pub fn thumbnail_url_fmt(&self) -> String { - get!( - self, - thumbnail_url, - "thumbnail URL", - (|url: &Url| url.to_string()) - ) - } - - #[must_use] - pub fn title_fmt_no_color(&self) -> String { - self.title.replace(['"', '„', '”', '“'], "'") - } - #[must_use] - pub fn title_fmt(&self, app: &App) -> String { - let title = self.title_fmt_no_color(); - maybe_add_color(app, title, |v| v.green().bold().to_string()) - } - - #[must_use] - pub fn url_fmt_no_color(&self) -> String { - self.url.as_str().replace('"', "\\\"") - } - #[must_use] - pub fn url_fmt(&self, app: &App) -> String { - let url = self.url_fmt_no_color(); - maybe_add_color(app, url, |v| v.italic().to_string()) - } - - pub async fn video_options_fmt_no_color(&self, app: &App) -> Result<String> { - let video_options = { - let opts = get_video_opts(app, &self.extractor_hash) - .await - .with_context(|| { - format!("Failed to get video options for video: '{}'", self.title) - })? - .to_cli_flags(app); - let opts_white = if opts.is_empty() { "" } else { " " }; - format!("{opts_white}{opts}") - }; - Ok(video_options) - } - pub async fn video_options_fmt(&self, app: &App) -> Result<String> { - let opts = self.video_options_fmt_no_color(app).await?; - Ok(maybe_add_color(app, opts, |v| v.bright_green().to_string())) - } -} diff --git a/crates/yt/src/videos/format_video.rs b/crates/yt/src/videos/format_video.rs new file mode 100644 index 0000000..6598780 --- /dev/null +++ b/crates/yt/src/videos/format_video.rs @@ -0,0 +1,133 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use colors::Colorize; + +use crate::{app::App, output::format_text, storage::db::video::Video, videos::RenderWithApp}; + +impl Video { + pub(crate) async fn to_info_display( + &self, + app: &App, + format: Option<String>, + ) -> Result<String> { + let cache_path = self.cache_path_fmt().to_string(app); + let description = self.description_fmt().to_string(app); + let duration = self.duration_fmt().to_string(app); + let extractor_hash = self.extractor_hash_fmt(app).await?.to_string(app); + let in_playlist = self.in_playlist_fmt().to_string(app); + let last_status_change = self.last_status_change_fmt().to_string(app); + let parent_subscription_name = self.parent_subscription_name_fmt().to_string(app); + let priority = self.priority_fmt().to_string(app); + let publish_date = self.publish_date_fmt().to_string(app); + let status = self.status_fmt().to_string(app); + let thumbnail_url = self.thumbnail_url_fmt().to_string(app); + let title = self.title_fmt().to_string(app); + let url = self.url_fmt().to_string(app); + let video_options = self.video_options_fmt(app).to_string(app); + + let watched_percentage_fmt = { + if let Some(percent) = self.watch_progress_percent_fmt() { + format!(" (watched: {})", percent.to_string(app)) + } else { + format!(" {}", self.watch_progress_fmt().to_string(app)) + } + }; + + let options = video_options.to_string(); + let options = options.trim(); + let description = format_text(description.to_string().as_str(), None); + + let string = if let Some(format) = format { + format + .replace("{title}", &title) + .replace("{extractor_hash}", &extractor_hash) + .replace("{cache_path}", &cache_path) + .replace("{duration}", &duration) + .replace("{watched_percentage_fmt}", &watched_percentage_fmt) + .replace("{parent_subscription_name}", &parent_subscription_name) + .replace("{priority}", &priority) + .replace("{publish_date}", &publish_date) + .replace("{status}", &status) + .replace("{last_status_change}", &last_status_change) + .replace("{in_playlist}", &in_playlist) + .replace("{thumbnail_url}", &thumbnail_url) + .replace("{url}", &url) + .replace("{options}", options) + .replace("{description}", &description) + } else { + format!( + "\ +{title} ({extractor_hash}) +| -> {cache_path} +| -> {duration}{watched_percentage_fmt} +| -> {parent_subscription_name} +| -> priority: {priority} +| -> {publish_date} +| -> status: {status} since {last_status_change} ({in_playlist}) +| -> {thumbnail_url} +| -> {url} +| -> options: {options} +{description}\n", + ) + }; + Ok(string) + } + + pub(crate) async fn to_line_display( + &self, + app: &App, + format: Option<String>, + ) -> Result<String> { + let status = self.status_fmt().to_string(app); + let extractor_hash = self.extractor_hash_fmt(app).await?.to_string(app); + let title = self.title_fmt().to_string(app); + let publish_date = self.publish_date_fmt().to_string(app); + let parent_subscription_name = self.parent_subscription_name_fmt().to_string(app); + let duration = self.duration_fmt().to_string(app); + let url = self.url_fmt().to_string(app); + + let f = if let Some(format) = format { + format + .replace("{status}", &status) + .replace("{extractor_hash}", &extractor_hash) + .replace("{title}", &title) + .replace("{publish_date}", &publish_date) + .replace("{parent_subscription_name}", &parent_subscription_name) + .replace("{duration}", &duration) + .replace("{url}", &url) + } else { + format!( + "{status} {extractor_hash} {title} {publish_date} {parent_subscription_name} {duration}" + ) + }; + + Ok(f) + } + + pub(crate) async fn to_select_file_display(&self, app: &App) -> Result<String> { + let f = format!( + r#"{}{} {} "{}" "{}" "{}" "{}" "{}"{}"#, + self.status_fmt().render(false), + self.video_options_fmt(app).render(false), + self.extractor_hash_fmt(app).await?.render(false), + self.title_fmt().render(false), + self.publish_date_fmt().render(false), + self.parent_subscription_name_fmt().render(false), + self.duration_fmt().render(false), + self.url_fmt().render(false), + '\n' + ); + + Ok(f) + } +} diff --git a/crates/yt/src/videos/mod.rs b/crates/yt/src/videos/mod.rs index e821772..c2f01fa 100644 --- a/crates/yt/src/videos/mod.rs +++ b/crates/yt/src/videos/mod.rs @@ -9,59 +9,205 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use anyhow::Result; -use futures::{TryStreamExt, stream::FuturesUnordered}; -use nucleo_matcher::{ - Matcher, - pattern::{CaseMatching, Normalization, Pattern}, -}; +use std::fmt::Write; -pub mod display; +use anyhow::{Context, Result}; +use colors::{Colorize, IntoCanvas}; +use url::Url; use crate::{ app::App, - storage::video_database::{Video, VideoStatusMarker, get}, + select::duration::MaybeDuration, + storage::db::video::{TimeStamp, Video, VideoStatus}, }; -async fn to_line_display_owned(video: Video, app: &App) -> Result<String> { - video.to_line_display(app).await +pub(crate) mod format_video; + +macro_rules! get { + ($value:expr, $key:ident, $name:expr, $code:tt) => { + if let Some(value) = &$value.$key { + $code(value) + } else { + concat!("[No ", $name, "]").to_owned() + } + }; +} + +pub(crate) trait RenderWithApp: Colorize { + fn to_string(self, app: &App) -> String { + self.render(app.config.global.display_colors) + } } +impl<C: Colorize> RenderWithApp for C {} + +impl Video { + #[must_use] + pub(crate) fn cache_path_fmt(&self) -> impl Colorize { + let cache_path = if let VideoStatus::Cached { + cache_path, + is_focused: _, + } = &self.status + { + cache_path.to_string_lossy().to_string() + } else { + "[No Cache Path]".to_owned() + }; + + cache_path.blue().bold() + } + + #[must_use] + pub(crate) fn description_fmt(&self) -> impl Colorize { + get!( + self, + description, + "Description", + (|value: &str| value.to_owned()) + ) + .into_canvas() + } + + #[must_use] + pub(crate) fn duration_fmt(&self) -> impl Colorize { + self.duration.cyan().bold() + } + + #[must_use] + pub(crate) fn watch_progress_fmt(&self) -> impl Colorize { + MaybeDuration::from_std(self.watch_progress).cyan().bold() + } + #[must_use] + pub(crate) fn watch_progress_percent_fmt(&self) -> Option<impl Colorize> { + self.duration.as_secs_f64().map(|duration| { + let watch_progress = self.watch_progress.as_secs_f64(); -pub async fn query(app: &App, limit: Option<usize>, search_query: Option<String>) -> Result<()> { - let all_videos = get::videos(app, VideoStatusMarker::ALL).await?; + (format!("{:0.0}%", (watch_progress / duration) * 100.0)).into_canvas() + }) + } + + pub(crate) async fn extractor_hash_fmt(&self, app: &App) -> Result<impl Colorize> { + let hash = self + .extractor_hash + .as_short_hash(app) + .await + .with_context(|| { + format!( + "Failed to format extractor hash, whilst formatting video: '{}'", + self.title + ) + })?; - // turn one video to a color display, to pre-warm the hash shrinking cache - if let Some(val) = all_videos.first() { - val.to_line_display(app).await?; + Ok(hash.purple().bold().italic()) } - let limit = limit.unwrap_or(all_videos.len()); + #[must_use] + pub(crate) fn in_playlist_fmt(&self) -> impl Colorize { + let output = match &self.status { + VideoStatus::Pick + | VideoStatus::Watch + | VideoStatus::Watched + | VideoStatus::Drop + | VideoStatus::Dropped => "Not in the playlist", + VideoStatus::Cached { is_focused, .. } => { + if *is_focused { + "In the playlist and focused" + } else { + "In the playlist" + } + } + }; + output.yellow().italic() + } + #[must_use] + pub(crate) fn last_status_change_fmt(&self) -> impl Colorize { + self.last_status_change.bright_cyan() + } - let all_video_strings: Vec<String> = all_videos - .into_iter() - .take(limit) - .map(|vid| to_line_display_owned(vid, app)) - .collect::<FuturesUnordered<_>>() - .try_collect::<Vec<String>>() - .await?; + #[must_use] + pub(crate) fn parent_subscription_name_fmt(&self) -> impl Colorize { + let psn = get!( + self, + parent_subscription_name, + "author", + (|sub: &str| sub.replace('"', "'")) + ); - if let Some(query) = search_query { - let mut matcher = Matcher::new(nucleo_matcher::Config::DEFAULT.match_paths()); + psn.bright_magenta() + } - let pattern_matches = Pattern::parse( - &query.replace(' ', "\\ "), - CaseMatching::Ignore, - Normalization::Smart, + #[must_use] + pub(crate) fn priority_fmt(&self) -> impl Colorize { + self.priority.into_canvas() + } + + #[must_use] + pub(crate) fn publish_date_fmt(&self) -> impl Colorize { + let date = get!( + self, + publish_date, + "release date", + (|date: &TimeStamp| date.to_string()) + ); + + date.bright_white().bold() + } + + #[must_use] + pub(crate) fn status_fmt(&self) -> impl Colorize { + // TODO: We might support `.trim()`ing that, as the extra whitespace could be bad in the + // selection file. <2024-10-07> + let status = self.status.as_marker().as_command().to_owned(); + + status.red().bold() + } + + #[must_use] + pub(crate) fn thumbnail_url_fmt(&self) -> impl Colorize { + get!( + self, + thumbnail_url, + "thumbnail URL", + (|url: &Url| url.to_string()) ) - .match_list(all_video_strings, &mut matcher); + .into_canvas() + } + + #[must_use] + pub(crate) fn title_fmt(&self) -> impl Colorize { + let title = self.title.replace(['"', '„', '”', '“'], "'"); - pattern_matches - .iter() - .rev() - .for_each(|(val, key)| println!("{val} ({key})")); - } else { - println!("{}", all_video_strings.join("\n")); + title.green().bold() } - Ok(()) + #[must_use] + pub(crate) fn url_fmt(&self) -> impl Colorize { + let url = self.url.as_str().replace('"', "\\\""); + + url.italic() + } + + pub(crate) fn video_options_fmt(&self, app: &App) -> impl Colorize { + let video_options = { + let mut opts = String::new(); + + if let Some(playback_speed) = self.playback_speed { + if (playback_speed - app.config.select.playback_speed).abs() > f64::EPSILON { + write!(opts, " --playback-speed '{}'", playback_speed).expect("In-memory"); + } + } + + if let Some(subtitle_langs) = &self.subtitle_langs { + if subtitle_langs != &app.config.select.subtitle_langs { + write!(opts, " --subtitle-langs '{}'", subtitle_langs).expect("In-memory"); + } + } + + let opts = opts.trim().to_owned(); + + let opts_white = if opts.is_empty() { "" } else { " " }; + format!("{opts_white}{opts}") + }; + + video_options.bright_green() + } } diff --git a/crates/yt/src/watch/mod.rs b/crates/yt/src/watch/mod.rs deleted file mode 100644 index c32a76f..0000000 --- a/crates/yt/src/watch/mod.rs +++ /dev/null @@ -1,178 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{ - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Duration, -}; - -use anyhow::{Context, Result}; -use libmpv2::{Mpv, events::EventContext}; -use log::{debug, info, trace, warn}; -use playlist_handler::{reload_mpv_playlist, save_watch_progress}; -use tokio::{task, time::sleep}; - -use self::playlist_handler::Status; -use crate::{ - app::App, - cache::maintain, - storage::video_database::{get, notify::wait_for_db_write}, -}; - -pub mod playlist; -pub mod playlist_handler; - -fn init_mpv(app: &App) -> Result<(Mpv, EventContext)> { - // set some default values, to make things easier (these can be overridden by the config file, - // which we load later) - let mpv = Mpv::with_initializer(|mpv| { - // Enable default key bindings, so the user can actually interact with - // the player (and e.g. close the window). - mpv.set_property("input-default-bindings", "yes")?; - mpv.set_property("input-vo-keyboard", "yes")?; - - // Show the on screen controller. - mpv.set_property("osc", "yes")?; - - // Don't automatically advance to the next video (or exit the player) - mpv.set_option("keep-open", "always")?; - - // Always display an window, even for non-video playback. - // As mpv does not have cli access, no window means no control and no user feedback. - mpv.set_option("force-window", "yes")?; - Ok(()) - }) - .context("Failed to initialize mpv")?; - - let config_path = &app.config.paths.mpv_config_path; - if config_path.try_exists()? { - info!("Found mpv.conf at '{}'!", config_path.display()); - mpv.command( - "load-config-file", - &[config_path - .to_str() - .context("Failed to parse the config path is utf8-stringt")?], - )?; - } else { - warn!( - "Did not find a mpv.conf file at '{}'", - config_path.display() - ); - } - - let input_path = &app.config.paths.mpv_input_path; - if input_path.try_exists()? { - info!("Found mpv.input.conf at '{}'!", input_path.display()); - mpv.command( - "load-input-conf", - &[input_path - .to_str() - .context("Failed to parse the input path as utf8 string")?], - )?; - } else { - warn!( - "Did not find a mpv.input.conf file at '{}'", - input_path.display() - ); - } - - let ev_ctx = EventContext::new(mpv.ctx); - ev_ctx.disable_deprecated_events()?; - - Ok((mpv, ev_ctx)) -} - -pub async fn watch(app: Arc<App>) -> Result<()> { - maintain(&app, false).await?; - - let (mpv, mut ev_ctx) = init_mpv(&app).context("Failed to initialize mpv instance")?; - let mpv = Arc::new(mpv); - reload_mpv_playlist(&app, &mpv, None, None).await?; - - let should_break = Arc::new(AtomicBool::new(false)); - - let local_app = Arc::clone(&app); - let local_mpv = Arc::clone(&mpv); - let local_should_break = Arc::clone(&should_break); - let progress_handle = task::spawn(async move { - loop { - if local_should_break.load(Ordering::Relaxed) { - break; - } - - if get::currently_focused_video(&local_app).await?.is_some() { - save_watch_progress(&local_app, &local_mpv).await?; - } - - sleep(Duration::from_secs(30)).await; - } - - Ok::<(), anyhow::Error>(()) - }); - - let mut have_warned = (false, 0); - 'watchloop: loop { - 'waitloop: while let Ok(value) = playlist_handler::status(&app).await { - match value { - Status::NoMoreAvailable => { - break 'watchloop; - } - Status::NoCached { marked_watch } => { - // try again next time. - if have_warned.0 { - if have_warned.1 != marked_watch { - warn!("Now {} videos are marked as to be watched.", marked_watch); - have_warned.1 = marked_watch; - } - } else { - warn!( - "There is nothing to watch yet, but still {} videos marked as to be watched. \ - Will idle, until they become available", - marked_watch - ); - have_warned = (true, marked_watch); - } - wait_for_db_write(&app).await?; - } - Status::Available { newly_available } => { - debug!("Check and found {newly_available} videos!"); - have_warned.0 = false; - - // Something just became available! - break 'waitloop; - } - } - } - - if let Some(ev) = ev_ctx.wait_event(30.) { - match ev { - Ok(event) => { - trace!("Mpv event triggered: {:#?}", event); - if playlist_handler::handle_mpv_event(&app, &mpv, &event) - .await - .with_context(|| format!("Failed to handle mpv event: '{event:#?}'"))? - { - break; - } - } - Err(e) => debug!("Mpv Event errored: {}", e), - } - } - } - - should_break.store(true, Ordering::Relaxed); - progress_handle.await??; - - Ok(()) -} diff --git a/crates/yt/src/watch/playlist.rs b/crates/yt/src/watch/playlist.rs deleted file mode 100644 index ff383d0..0000000 --- a/crates/yt/src/watch/playlist.rs +++ /dev/null @@ -1,99 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::path::Path; - -use crate::{ - ansi_escape_codes::{cursor_up, erase_in_display_from_cursor}, - app::App, - storage::video_database::{Video, VideoStatus, get, notify::wait_for_db_write}, -}; - -use anyhow::Result; -use futures::{TryStreamExt, stream::FuturesOrdered}; - -/// Extract the values of the [`VideoStatus::Cached`] value from a Video. -fn cache_values(video: &Video) -> (&Path, bool) { - if let VideoStatus::Cached { - cache_path, - is_focused, - } = &video.status - { - (cache_path, *is_focused) - } else { - unreachable!("All of these videos should be cached"); - } -} - -/// # Panics -/// Only if internal assertions fail. -pub async fn playlist(app: &App, watch: bool) -> Result<()> { - let mut previous_output_length = 0; - loop { - let playlist = get::playlist(app).await?.to_videos(); - - let output = playlist - .into_iter() - .map(|video| async move { - let mut output = String::new(); - - let (_, is_focused) = cache_values(&video); - - if is_focused { - output.push_str("🔻 "); - } else { - output.push_str(" "); - } - - output.push_str(&video.title_fmt(app)); - - output.push_str(" ("); - output.push_str(&video.parent_subscription_name_fmt(app)); - output.push(')'); - - output.push_str(" ["); - output.push_str(&video.duration_fmt(app)); - - if is_focused { - output.push_str(" ("); - output.push_str(&if let Some(duration) = video.duration.as_secs() { - format!("{:0.0}%", (video.watch_progress.as_secs() / duration) * 100) - } else { - video.watch_progress_fmt(app) - }); - output.push(')'); - } - output.push(']'); - - output.push('\n'); - - Ok::<String, anyhow::Error>(output) - }) - .collect::<FuturesOrdered<_>>() - .try_collect::<String>() - .await?; - - // Delete the previous output - cursor_up(previous_output_length); - erase_in_display_from_cursor(); - - previous_output_length = output.chars().filter(|ch| *ch == '\n').count(); - - print!("{output}"); - - if !watch { - break; - } - - wait_for_db_write(app).await?; - } - - Ok(()) -} diff --git a/crates/yt/src/watch/playlist_handler/mod.rs b/crates/yt/src/watch/playlist_handler/mod.rs deleted file mode 100644 index 29b8f39..0000000 --- a/crates/yt/src/watch/playlist_handler/mod.rs +++ /dev/null @@ -1,342 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::{cmp::Ordering, time::Duration}; - -use crate::{ - app::App, - storage::video_database::{ - VideoStatus, VideoStatusMarker, - extractor_hash::ExtractorHash, - get::{self, Playlist, PlaylistIndex}, - set, - }, -}; - -use anyhow::{Context, Result}; -use libmpv2::{EndFileReason, Mpv, events::Event}; -use log::{debug, info}; - -mod client_messages; - -#[derive(Debug, Clone, Copy)] -pub enum Status { - /// There are no videos cached and no more marked to be watched. - /// Waiting is pointless. - NoMoreAvailable, - - /// There are no videos cached, but some (> 0) are marked to be watched. - /// So we should wait for them to become available. - NoCached { marked_watch: usize }, - - /// There are videos cached and ready to be inserted into the playback queue. - Available { newly_available: usize }, -} - -fn mpv_message(mpv: &Mpv, message: &str, time: Duration) -> Result<()> { - mpv.command( - "show-text", - &[message, time.as_millis().to_string().as_str()], - )?; - Ok(()) -} - -async fn apply_video_options(app: &App, mpv: &Mpv, video: &ExtractorHash) -> Result<()> { - let options = get::video_mpv_opts(app, video).await?; - let video = get::video_by_hash(app, video).await?; - - mpv.set_property("speed", options.playback_speed)?; - - // We already start at 0, so setting it twice adds a uncomfortable skip sound. - if video.watch_progress.as_secs() != 0 { - mpv.set_property( - "time-pos", - i64::try_from(video.watch_progress.as_secs()).expect("This should not overflow"), - )?; - } - Ok(()) -} - -async fn mark_video_watched(app: &App, mpv: &Mpv) -> Result<()> { - let current_video = get::currently_focused_video(app) - .await? - .expect("This should be some at this point"); - - debug!( - "playlist handler will mark video '{}' watched.", - current_video.title - ); - - save_watch_progress(app, mpv).await?; - - set::video_watched(app, ¤t_video.extractor_hash).await?; - - Ok(()) -} - -/// Saves the `watch_progress` of the currently focused video. -pub(super) async fn save_watch_progress(app: &App, mpv: &Mpv) -> Result<()> { - let current_video = get::currently_focused_video(app) - .await? - .expect("This should be some at this point"); - let watch_progress = u32::try_from( - mpv.get_property::<i64>("time-pos") - .context("Failed to get the watchprogress of the currently playling video")?, - ) - .expect("This conversion should never fail as the `time-pos` property is positive"); - - debug!( - "Setting the watch progress for the current_video '{}' to {watch_progress}s", - current_video.title_fmt_no_color() - ); - - set::video_watch_progress(app, ¤t_video.extractor_hash, watch_progress).await -} - -/// Sync the mpv playlist with the internal playlist. -/// -/// This takes an `maybe_playlist` argument, if you have already fetched the playlist and want to -/// add that. -pub(super) async fn reload_mpv_playlist( - app: &App, - mpv: &Mpv, - maybe_playlist: Option<Playlist>, - maybe_index: Option<PlaylistIndex>, -) -> Result<()> { - fn get_playlist_count(mpv: &Mpv) -> Result<usize> { - mpv.get_property::<i64>("playlist/count") - .context("Failed to get mpv playlist len") - .map(|count| { - usize::try_from(count).expect("The playlist_count should always be positive") - }) - } - - if get_playlist_count(mpv)? != 0 { - // We could also use `loadlist`, but that would require use to start a unix socket or even - // write all the video paths to a file beforehand - mpv.command("playlist-clear", &[])?; - mpv.command("playlist-remove", &["current"])?; - } - - assert_eq!( - get_playlist_count(mpv)?, - 0, - "The playlist should be empty at this point." - ); - - let playlist = if let Some(p) = maybe_playlist { - p - } else { - get::playlist(app).await? - }; - - debug!("Will add {} videos to playlist.", playlist.len()); - playlist.into_iter().try_for_each(|cache_path| { - mpv.command( - "loadfile", - &[ - cache_path.to_str().with_context(|| { - format!( - "Failed to parse the video cache path ('{}') as valid utf8", - cache_path.display() - ) - })?, - "append-play", - ], - )?; - - Ok::<(), anyhow::Error>(()) - })?; - - let index = if let Some(index) = maybe_index { - let index = usize::from(index); - let playlist_length = get_playlist_count(mpv)?; - - match index.cmp(&playlist_length) { - Ordering::Greater => { - unreachable!( - "The index '{index}' execeeds the playlist length '{playlist_length}'." - ); - } - Ordering::Less => index, - Ordering::Equal => { - // The index is pointing to the end of the playlist. We could either go the second - // to last entry (i.e., one entry back) or wrap around to the start. - // We wrap around: - 0 - } - } - } else { - get::current_playlist_index(app) - .await? - .map_or(0, usize::from) - }; - mpv.set_property("playlist-pos", index.to_string().as_str())?; - - Ok(()) -} - -/// Return the status of the playback queue -pub async fn status(app: &App) -> Result<Status> { - let playlist = get::playlist(app).await?; - - let playlist_len = playlist.len(); - let marked_watch_num = get::videos(app, &[VideoStatusMarker::Watch]).await?.len(); - - if playlist_len == 0 && marked_watch_num == 0 { - Ok(Status::NoMoreAvailable) - } else if playlist_len == 0 && marked_watch_num != 0 { - Ok(Status::NoCached { - marked_watch: marked_watch_num, - }) - } else if playlist_len != 0 { - Ok(Status::Available { - newly_available: playlist_len, - }) - } else { - unreachable!( - "The playlist length is {playlist_len}, but the number of marked watch videos is {marked_watch_num}! This is a bug." - ); - } -} - -/// # Returns -/// This will return [`true`], if the event handling should be stopped -/// -/// # Panics -/// Only if internal assertions fail. -#[allow(clippy::too_many_lines)] -pub async fn handle_mpv_event(app: &App, mpv: &Mpv, event: &Event<'_>) -> Result<bool> { - match event { - Event::EndFile(r) => match r.reason { - EndFileReason::Eof => { - info!("Mpv reached the end of the current video. Marking it watched."); - mark_video_watched(app, mpv).await?; - reload_mpv_playlist(app, mpv, None, None).await?; - } - EndFileReason::Stop => { - // This reason is incredibly ambiguous. It _both_ means actually pausing a - // video and going to the next one in the playlist. - // Oh, and it's also called, when a video is removed from the playlist (at - // least via "playlist-remove current") - info!("Paused video (or went to next playlist entry); Doing nothing"); - } - EndFileReason::Quit => { - info!("Mpv quit. Exiting playback"); - - save_watch_progress(app, mpv).await?; - - return Ok(true); - } - EndFileReason::Error => { - unreachable!("This should have been raised as a separate error") - } - EndFileReason::Redirect => { - // TODO: We probably need to handle this somehow <2025-02-17> - } - }, - Event::StartFile(_) => { - let mpv_pos = usize::try_from(mpv.get_property::<i64>("playlist-pos")?) - .expect("The value is strictly positive"); - - let next_video = { - let yt_pos = get::current_playlist_index(app).await?.map(usize::from); - - if (Some(mpv_pos) != yt_pos) || yt_pos.is_none() { - let playlist = get::playlist(app).await?; - let video = playlist - .get(PlaylistIndex::from(mpv_pos)) - .expect("The mpv pos should not be out of bounds"); - - set::focused( - app, - &video.extractor_hash, - get::currently_focused_video(app) - .await? - .as_ref() - .map(|v| &v.extractor_hash), - ) - .await?; - - video.extractor_hash - } else { - get::currently_focused_video(app) - .await? - .expect("We have a focused video") - .extractor_hash - } - }; - - apply_video_options(app, mpv, &next_video).await?; - } - Event::Seek => { - save_watch_progress(app, mpv).await?; - } - Event::ClientMessage(a) => { - debug!("Got Client Message event: '{}'", a.join(" ")); - - match a.as_slice() { - &["yt-comments-external"] => { - client_messages::handle_yt_comments_external(app).await?; - } - &["yt-comments-local"] => { - client_messages::handle_yt_comments_local(app, mpv).await?; - } - - &["yt-description-external"] => { - client_messages::handle_yt_description_external(app).await?; - } - &["yt-description-local"] => { - client_messages::handle_yt_description_local(app, mpv).await?; - } - - &["yt-mark-picked"] => { - let current_video = get::currently_focused_video(app) - .await? - .expect("This should exist at this point"); - let current_index = get::current_playlist_index(app) - .await? - .expect("This should exist, as we can mark this video picked"); - - save_watch_progress(app, mpv).await?; - - set::video_status( - app, - ¤t_video.extractor_hash, - VideoStatus::Pick, - Some(current_video.priority), - ) - .await?; - - reload_mpv_playlist(app, mpv, None, Some(current_index)).await?; - mpv_message(mpv, "Marked the video as picked", Duration::from_secs(3))?; - } - &["yt-mark-watched"] => { - let current_index = get::current_playlist_index(app) - .await? - .expect("This should exist, as we can mark this video picked"); - mark_video_watched(app, mpv).await?; - - reload_mpv_playlist(app, mpv, None, Some(current_index)).await?; - mpv_message(mpv, "Marked the video watched", Duration::from_secs(3))?; - } - &["yt-check-new-videos"] => { - reload_mpv_playlist(app, mpv, None, None).await?; - } - other => { - debug!("Unknown message: {}", other.join(" ")); - } - } - } - _ => {} - } - - Ok(false) -} diff --git a/crates/yt/src/yt_dlp/mod.rs b/crates/yt/src/yt_dlp/mod.rs new file mode 100644 index 0000000..eaa80a1 --- /dev/null +++ b/crates/yt/src/yt_dlp/mod.rs @@ -0,0 +1,253 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{borrow::ToOwned, str::FromStr, time::Duration}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use futures::{FutureExt, future::BoxFuture}; +use log::{error, warn}; +use serde_json::json; +use tokio::{fs, io}; +use url::Url; +use yt_dlp::{ + YoutubeDL, info_json::InfoJson, json_cast, json_get, json_try_get, options::YoutubeDLOptions, +}; + +use crate::{ + app::App, + select::duration::MaybeDuration, + shared::bytes::Bytes, + storage::db::{ + extractor_hash::ExtractorHash, + subscription::Subscription, + video::{Priority, TimeStamp, Video, VideoStatus}, + }, +}; + +pub(crate) fn yt_dlp_opts_updating(max_backlog: usize) -> Result<YoutubeDL> { + Ok(YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", max_backlog) + .set("noplaylist", false) + .set( + "extractor_args", + json! {{"youtubetab": {"approximate_date": [""]}}}, + ) + // // TODO: This also removes unlisted and other stuff. Find a good way to remove the + // // members-only videos from the feed. <2025-04-17> + // .set("match-filter", "availability=public") + .build()?) +} + +impl Video { + pub(crate) fn get_approx_size(&self) -> Result<u64> { + let yt_dlp = { + YoutubeDLOptions::new() + .set("prefer_free_formats", true) + .set("format", "bestvideo[height<=?1080]+bestaudio/best") + .set("fragment_retries", 10) + .set("retries", 10) + .set("getcomments", false) + .set("ignoreerrors", false) + .build() + .context("Failed to instanciate get approx size yt_dlp") + }?; + + let result = yt_dlp + .extract_info(&self.url, false, true) + .with_context(|| format!("Failed to extract video information: '{}'", self.title))?; + + let size = if let Some(filesize) = json_try_get!(result, "filesize", as_u64) { + filesize + } else if let Some(num) = json_try_get!(result, "filesize_approx", as_u64) { + num + } else if let Some(duration) = json_try_get!(result, "duration", as_f64) + && let Some(tbr) = json_try_get!(result, "tbr", as_f64) + { + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let duration = duration.ceil() as u64; + + // TODO: yt_dlp gets this from the format + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let tbr = tbr.ceil() as u64; + + duration * tbr * (1000 / 8) + } else { + let hardcoded_default = Bytes::from_str("250 MiB").expect("This is hardcoded"); + error!( + "Failed to find a filesize for video: {:?} (Using hardcoded value of {})", + self.title, hardcoded_default + ); + hardcoded_default.as_u64() + }; + + Ok(size) + } +} + +impl Video { + #[allow(clippy::too_many_lines)] + pub(crate) fn from_info_json(entry: &InfoJson, sub: Option<&Subscription>) -> Result<Video> { + fn fmt_context(date: &str, extended: Option<&str>) -> String { + let f = format!( + "Failed to parse the `upload_date` of the entry ('{date}'). \ + Expected `YYYY-MM-DD`, has the format changed?" + ); + if let Some(date_string) = extended { + format!("{f}\nThe parsed '{date_string}' can't be turned to a valid UTC date.'") + } else { + f + } + } + + let publish_date = if let Some(date) = json_try_get!(entry, "upload_date", as_str) { + let year: u32 = date + .chars() + .take(4) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let month: u32 = date + .chars() + .skip(4) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let day: u32 = date + .chars() + .skip(4 + 2) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + + let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z"); + Some( + DateTime::<Utc>::from_str(&date_string) + .with_context(|| fmt_context(date, Some(&date_string)))? + .timestamp(), + ) + } else { + warn!( + "The video '{}' lacks it's upload date!", + json_get!(entry, "title", as_str) + ); + None + }; + + let thumbnail_url = match ( + &json_try_get!(entry, "thumbnails", as_array), + &json_try_get!(entry, "thumbnail", as_str), + ) { + (None, None) => None, + (None, Some(thumbnail)) => Some(Url::from_str(thumbnail)?), + + // TODO: The algorithm is not exactly the best <2024-05-28> + (Some(thumbnails), None) => { + if let Some(thumbnail) = thumbnails.first() { + Some(Url::from_str(json_get!( + json_cast!(thumbnail, as_object), + "url", + as_str + ))?) + } else { + None + } + } + (Some(_), Some(thumnail)) => Some(Url::from_str(thumnail)?), + }; + + let url = { + let smug_url: Url = json_get!(entry, "webpage_url", as_str).parse()?; + // TODO(@bpeetz): We should probably add this? <2025-06-14> + // if '#__youtubedl_smuggle' not in smug_url: + // return smug_url, default + // url, _, sdata = smug_url.rpartition('#') + // jsond = urllib.parse.parse_qs(sdata)['__youtubedl_smuggle'][0] + // data = json.loads(jsond) + // return url, data + + smug_url + }; + + let extractor_hash = ExtractorHash::from_info_json(entry); + + let subscription_name = if let Some(sub) = sub { + Some(sub.name.clone()) + } else if let Some(uploader) = json_try_get!(entry, "uploader", as_str) { + if json_try_get!(entry, "webpage_url_domain", as_str) == Some("youtube.com") { + Some(format!("{uploader} - Videos")) + } else { + Some(uploader.to_owned()) + } + } else { + None + }; + + let video = Video { + description: json_try_get!(entry, "description", as_str).map(ToOwned::to_owned), + duration: MaybeDuration::from_maybe_secs_f64(json_try_get!(entry, "duration", as_f64)), + extractor_hash, + last_status_change: TimeStamp::from_now(), + parent_subscription_name: subscription_name, + priority: Priority::default(), + publish_date: publish_date.map(TimeStamp::from_secs), + status: VideoStatus::Pick, + thumbnail_url, + title: json_get!(entry, "title", as_str).to_owned(), + url, + watch_progress: Duration::default(), + playback_speed: None, + subtitle_langs: None, + }; + Ok(video) + } +} + +pub(crate) async fn get_current_cache_allocation(app: &App) -> Result<Bytes> { + fn dir_size(mut dir: fs::ReadDir) -> BoxFuture<'static, Result<Bytes>> { + async move { + let mut acc = 0; + while let Some(entry) = dir.next_entry().await? { + let size = match entry.metadata().await? { + data if data.is_dir() => { + let path = entry.path(); + let read_dir = fs::read_dir(path).await?; + + dir_size(read_dir).await?.as_u64() + } + data => data.len(), + }; + acc += size; + } + Ok(Bytes::new(acc)) + } + .boxed() + } + + let read_dir_result = match fs::read_dir(&app.config.paths.download_dir).await { + Ok(ok) => ok, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => { + unreachable!("The download dir should always be created in the config finalizers."); + } + err => Err(io::Error::from(err)).with_context(|| { + format!( + "Failed to get dir size of download dir at: '{}'", + &app.config.paths.download_dir.display() + ) + })?, + }, + }; + + dir_size(read_dir_result).await +} diff --git a/crates/yt/tests/_testenv/init.rs b/crates/yt/tests/_testenv/init.rs new file mode 100644 index 0000000..5970c7c --- /dev/null +++ b/crates/yt/tests/_testenv/init.rs @@ -0,0 +1,136 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + env, + fs::{self, OpenOptions}, + io::{self, Write}, + path::PathBuf, +}; + +use crate::{_testenv::Paths, testenv::TestEnv}; + +fn target_dir() -> PathBuf { + // Tests exe is in target/debug/deps, the *yt* exe is in target/debug + env::current_exe() + .expect("./target/debug/deps/yt-*") + .parent() + .expect("./target/debug/deps") + .parent() + .expect("./target/debug") + .parent() + .expect("./target") + .to_path_buf() +} + +fn test_dir(name: &'static str) -> PathBuf { + target_dir().join("tests").join(name) +} + +fn prepare_files_and_dirs(name: &'static str) -> io::Result<Paths> { + let test_dir = test_dir(name); + + fs::create_dir_all(&test_dir)?; + + let db_path = test_dir.join("database.sqlite"); + let last_selection_path = test_dir.join("last_selection"); + let config_path = test_dir.join("config.toml"); + let download_dir = test_dir.join("download"); + + { + // Remove all files, that are not the download dir. + for entry in fs::read_dir(test_dir).expect("Works") { + let entry = entry.unwrap(); + let entry_ft = entry.file_type().unwrap(); + + if entry_ft.is_dir() && entry.file_name() == "download" { + continue; + } + + if entry_ft.is_dir() { + fs::remove_dir_all(entry.path()).unwrap(); + } else { + fs::remove_file(entry.path()).unwrap(); + } + } + } + + fs::create_dir_all(&download_dir)?; + + { + let mut config_file = OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(&config_path)?; + + writeln!( + config_file, + r#"[paths] +download_dir = "{}" +mpv_config_path = "/dev/null" +mpv_input_path = "/dev/null" +database_path = "{}" +last_selection_path = "{}" + +[download] +max_cache_size = "100GiB" + +[update] +max_backlog = 1 +"#, + download_dir.display(), + db_path.display(), + last_selection_path.display(), + )?; + config_file.flush()?; + } + + Ok(Paths { + db: db_path, + last_selection: last_selection_path, + config: config_path, + download_dir, + }) +} + +/// Find the *yt* executable. +fn find_yt_exe() -> PathBuf { + let target = target_dir().join("debug"); + + let exe_name = if cfg!(windows) { "yt.exe" } else { "yt" }; + + target.join(exe_name) +} + +impl TestEnv { + pub(crate) fn new(name: &'static str) -> TestEnv { + let yt_exe = find_yt_exe(); + let test_dir = test_dir(name); + + let paths = prepare_files_and_dirs(name).expect("config dir"); + + TestEnv { + name, + yt_exe, + test_dir, + paths, + spawned_childs: vec![], + } + } +} + +impl Drop for TestEnv { + fn drop(&mut self) { + for child in &mut self.spawned_childs { + drop(child.kill()); + } + } +} diff --git a/crates/yt/tests/_testenv/mod.rs b/crates/yt/tests/_testenv/mod.rs new file mode 100644 index 0000000..38d1f0a --- /dev/null +++ b/crates/yt/tests/_testenv/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! This code was taken from *fd* at 30-06-2025. + +use std::{path::PathBuf, process}; + +mod init; +mod run; +pub(crate) mod util; + +/// Environment for the integration tests. +pub(crate) struct TestEnv { + pub(crate) name: &'static str, + pub(crate) yt_exe: PathBuf, + pub(crate) test_dir: PathBuf, + + pub(crate) paths: Paths, + + pub(crate) spawned_childs: Vec<process::Child>, +} + +pub(crate) struct Paths { + pub(crate) db: PathBuf, + pub(crate) last_selection: PathBuf, + pub(crate) config: PathBuf, + pub(crate) download_dir: PathBuf, +} diff --git a/crates/yt/tests/_testenv/run.rs b/crates/yt/tests/_testenv/run.rs new file mode 100644 index 0000000..578d823 --- /dev/null +++ b/crates/yt/tests/_testenv/run.rs @@ -0,0 +1,183 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::HashMap, + process::{self, Stdio}, +}; + +use colors::{Colorize, IntoCanvas}; + +use crate::testenv::TestEnv; + +/// Format an error message for when *yt* did not exit successfully. +fn format_exit_error(args: &[&str], output: &process::Output) -> String { + format!( + "`yt {}` did not exit successfully.\nstdout:\n---\n{}---\nstderr:\n---\n{}---", + args.join(" "), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// Format an error message for when the output of *yt* did not match the expected output. +fn format_output_error(args: &[&str], expected: &str, actual: &str) -> String { + fn normalize_str(input: &str) -> &str { + if input.is_empty() { "<Empty>" } else { input } + } + + let expected = normalize_str(expected); + let actual = normalize_str(actual); + + format!( + concat!( + "`yt {}` did not produce the expected output.\n", + "expected:\n---\n{}\n---\n and actual:\n---\n{}\n---\n" + ), + args.join(" "), + expected, + actual + ) +} + +/// Normalize the output for comparison. +fn normalize_output(s: &str, trim_start: bool, normalize_line: bool) -> String { + // Split into lines and normalize separators. + let mut lines = s + .replace('\0', "NULL\n") + .lines() + .map(|line| { + let line = if trim_start { line.trim_start() } else { line }; + let line = line.replace('/', std::path::MAIN_SEPARATOR_STR); + if normalize_line { + let mut words: Vec<_> = line.split_whitespace().collect(); + words.sort_unstable(); + return words.join(" "); + } + line + }) + .collect::<Vec<_>>(); + + lines.sort(); + lines.join("\n") +} + +impl TestEnv { + /// Assert that calling *yt* in the specified path under the root working directory, + /// and with the specified arguments produces the expected output. + pub(crate) fn assert_output(&self, args: &[&str], expected: &str) { + let expected = normalize_output(expected, true, false); + let actual = self.run(args); + + assert!( + expected == actual, + "{}", + format_output_error(args, &expected, &actual) + ); + } + + /// Run *yt* once with the first args, pipe the output of this command to *yt* with the second + /// args and return the normalized output. + pub(crate) fn run_piped(&self, first_args: &[&str], second_args: &[&str]) -> String { + let mut first_cmd = self.prepare_yt(first_args); + let mut second_cmd = self.prepare_yt(second_args); + + first_cmd.stdout(Stdio::piped()); + let mut first_child = first_cmd.spawn().expect("yt spawn"); + + second_cmd.stdin(first_child.stdout.take().expect("Was set")); + + let first_output = first_child.wait_with_output().expect("yt run"); + assert!( + first_output.status.success(), + "{}", + format_exit_error(first_args, &first_output) + ); + + Self::finalize_cmd(second_cmd, second_args) + } + + /// Run *yt*, with the given args. + /// Returns the normalized stdout output. + pub(crate) fn run(&self, args: &[&str]) -> String { + let cmd = self.prepare_yt(args); + Self::finalize_cmd(cmd, args) + } + + /// Run *yt*, with the given args and environment variables. + /// Returns the normalized stdout output. + pub(crate) fn run_env(&self, args: &[&str], env: &HashMap<&str, &str>) -> String { + let mut cmd = self.prepare_yt(args); + cmd.envs(env.iter()); + Self::finalize_cmd(cmd, args) + } + + /// Run *yt*, with the given args and fork into the background. + /// Returns a sender for the lines on stdout. + pub(crate) fn run_background(&mut self, args: &[&str]) -> process::ChildStdout { + let mut cmd = self.prepare_yt(args); + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::inherit()); + + // The whole point of this function, is calling a command and keep it running for the whole + // programs run-time. + // + // And we provide a clean-up mechanism. + #[allow(clippy::zombie_processes)] + let mut child = cmd.spawn().expect("yt spawn"); + + let stdout = child.stdout.take().expect("Was piped"); + + self.spawned_childs.push(child); + + stdout + } + + fn finalize_cmd(mut cmd: process::Command, args: &[&str]) -> String { + let child = cmd.spawn().expect("yt spawn"); + let output = child.wait_with_output().expect("yt output"); + + assert!( + output.status.success(), + "{}", + format_exit_error(args, &output) + ); + + normalize_output(&String::from_utf8_lossy(&output.stdout), false, false) + } + + fn prepare_yt(&self, args: &[&str]) -> process::Command { + let mut cmd = process::Command::new(&self.yt_exe); + + cmd.current_dir(&self.test_dir); + + cmd.args([ + "-v", + "--color", + "false", + "--config-path", + self.paths.config.to_str().unwrap(), + ]); + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + cmd.args(args); + + eprintln!( + "{} `yt {}`", + self.name.blue().italic().render(true), + args.join(" ") + ); + + cmd + } +} diff --git a/crates/yt/tests/_testenv/util.rs b/crates/yt/tests/_testenv/util.rs new file mode 100644 index 0000000..6633fbf --- /dev/null +++ b/crates/yt/tests/_testenv/util.rs @@ -0,0 +1,371 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::_testenv::TestEnv; + +use std::{ + collections::HashMap, + fs::{OpenOptions, Permissions}, + io::Write, + os::unix::fs::PermissionsExt, + path::PathBuf, +}; + +pub(crate) fn get_first_hash(env: &TestEnv) -> String { + let output = env.run(&["videos", "ls", "--format", "{extractor_hash}"]); + + let first_hash = output.lines().next().unwrap(); + first_hash.to_owned() +} + +#[derive(Clone, Copy)] +pub(crate) enum Subscription { + Tagesschau, +} + +impl Subscription { + const fn as_url(self) -> &'static str { + match self { + Subscription::Tagesschau => { + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU" + } + } + } + + const fn as_name(self) -> &'static str { + match self { + Subscription::Tagesschau => "Tagesschau", + } + } +} + +/// Provide the given number of videos. +pub(crate) fn provide_videos(env: &TestEnv, sub: Subscription, num: u8) { + add_sub(env, sub); + update_sub(env, num, sub); +} + +pub(crate) fn add_sub(env: &TestEnv, sub: Subscription) { + env.run(&[ + "subs", + "add", + sub.as_url(), + "--name", + sub.as_name(), + "--no-check", + ]); +} + +pub(crate) fn update_sub(env: &TestEnv, num_of_videos: u8, sub: Subscription) { + let num_of_videos: &str = u8_to_char(num_of_videos); + + env.run(&["update", "--max-backlog", num_of_videos, sub.as_name()]); +} + +pub(crate) fn run_select(env: &TestEnv, sed_regex: &str) { + let mut map = HashMap::new(); + + let command = make_command( + env, + format!(r#"sed --in-place '{sed_regex}' "$1""#).as_str(), + ); + map.insert("EDITOR", command.to_str().unwrap()); + + env.run_env(&["select", "file", "--done"], &map); +} + +fn make_command(env: &TestEnv, shell_command: &str) -> PathBuf { + let command_path = env.test_dir.join("command.sh"); + + { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&command_path) + .unwrap(); + + writeln!(file, "#!/usr/bin/env sh").unwrap(); + + file.write_all(shell_command.as_bytes()).unwrap(); + file.flush().unwrap(); + + { + let perms = Permissions::from_mode(0o0700); + file.set_permissions(perms).unwrap(); + } + } + + command_path +} + +// Char conversion {{{ +#[allow(clippy::too_many_lines)] +const fn u8_to_char(input: u8) -> &'static str { + match input { + 0 => "0", + 1 => "1", + 2 => "2", + 3 => "3", + 4 => "4", + 5 => "5", + 6 => "6", + 7 => "7", + 8 => "8", + 9 => "9", + 10 => "10", + 11 => "11", + 12 => "12", + 13 => "13", + 14 => "14", + 15 => "15", + 16 => "16", + 17 => "17", + 18 => "18", + 19 => "19", + 20 => "20", + 21 => "21", + 22 => "22", + 23 => "23", + 24 => "24", + 25 => "25", + 26 => "26", + 27 => "27", + 28 => "28", + 29 => "29", + 30 => "30", + 31 => "31", + 32 => "32", + 33 => "33", + 34 => "34", + 35 => "35", + 36 => "36", + 37 => "37", + 38 => "38", + 39 => "39", + 40 => "40", + 41 => "41", + 42 => "42", + 43 => "43", + 44 => "44", + 45 => "45", + 46 => "46", + 47 => "47", + 48 => "48", + 49 => "49", + 50 => "50", + 51 => "51", + 52 => "52", + 53 => "53", + 54 => "54", + 55 => "55", + 56 => "56", + 57 => "57", + 58 => "58", + 59 => "59", + 60 => "60", + 61 => "61", + 62 => "62", + 63 => "63", + 64 => "64", + 65 => "65", + 66 => "66", + 67 => "67", + 68 => "68", + 69 => "69", + 70 => "70", + 71 => "71", + 72 => "72", + 73 => "73", + 74 => "74", + 75 => "75", + 76 => "76", + 77 => "77", + 78 => "78", + 79 => "79", + 80 => "80", + 81 => "81", + 82 => "82", + 83 => "83", + 84 => "84", + 85 => "85", + 86 => "86", + 87 => "87", + 88 => "88", + 89 => "89", + 90 => "90", + 91 => "91", + 92 => "92", + 93 => "93", + 94 => "94", + 95 => "95", + 96 => "96", + 97 => "97", + 98 => "98", + 99 => "99", + 100 => "100", + 101 => "101", + 102 => "102", + 103 => "103", + 104 => "104", + 105 => "105", + 106 => "106", + 107 => "107", + 108 => "108", + 109 => "109", + 110 => "110", + 111 => "111", + 112 => "112", + 113 => "113", + 114 => "114", + 115 => "115", + 116 => "116", + 117 => "117", + 118 => "118", + 119 => "119", + 120 => "120", + 121 => "121", + 122 => "122", + 123 => "123", + 124 => "124", + 125 => "125", + 126 => "126", + 127 => "127", + 128 => "128", + 129 => "129", + 130 => "130", + 131 => "131", + 132 => "132", + 133 => "133", + 134 => "134", + 135 => "135", + 136 => "136", + 137 => "137", + 138 => "138", + 139 => "139", + 140 => "140", + 141 => "141", + 142 => "142", + 143 => "143", + 144 => "144", + 145 => "145", + 146 => "146", + 147 => "147", + 148 => "148", + 149 => "149", + 150 => "150", + 151 => "151", + 152 => "152", + 153 => "153", + 154 => "154", + 155 => "155", + 156 => "156", + 157 => "157", + 158 => "158", + 159 => "159", + 160 => "160", + 161 => "161", + 162 => "162", + 163 => "163", + 164 => "164", + 165 => "165", + 166 => "166", + 167 => "167", + 168 => "168", + 169 => "169", + 170 => "170", + 171 => "171", + 172 => "172", + 173 => "173", + 174 => "174", + 175 => "175", + 176 => "176", + 177 => "177", + 178 => "178", + 179 => "179", + 180 => "180", + 181 => "181", + 182 => "182", + 183 => "183", + 184 => "184", + 185 => "185", + 186 => "186", + 187 => "187", + 188 => "188", + 189 => "189", + 190 => "190", + 191 => "191", + 192 => "192", + 193 => "193", + 194 => "194", + 195 => "195", + 196 => "196", + 197 => "197", + 198 => "198", + 199 => "199", + 200 => "200", + 201 => "201", + 202 => "202", + 203 => "203", + 204 => "204", + 205 => "205", + 206 => "206", + 207 => "207", + 208 => "208", + 209 => "209", + 210 => "210", + 211 => "211", + 212 => "212", + 213 => "213", + 214 => "214", + 215 => "215", + 216 => "216", + 217 => "217", + 218 => "218", + 219 => "219", + 220 => "220", + 221 => "221", + 222 => "222", + 223 => "223", + 224 => "224", + 225 => "225", + 226 => "226", + 227 => "227", + 228 => "228", + 229 => "229", + 230 => "230", + 231 => "231", + 232 => "232", + 233 => "233", + 234 => "234", + 235 => "235", + 236 => "236", + 237 => "237", + 238 => "238", + 239 => "239", + 240 => "240", + 241 => "241", + 242 => "242", + 243 => "243", + 244 => "244", + 245 => "245", + 246 => "246", + 247 => "247", + 248 => "248", + 249 => "249", + 250 => "250", + 251 => "251", + 252 => "252", + 253 => "253", + 254 => "254", + 255 => "255", + } +} +// }}} diff --git a/crates/yt/tests/select/base.rs b/crates/yt/tests/select/base.rs new file mode 100644 index 0000000..24e198b --- /dev/null +++ b/crates/yt/tests/select/base.rs @@ -0,0 +1,50 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, fs}; + +use crate::{ + _testenv::{ + TestEnv, + util::{self, Subscription}, + }, + select::get_videos_in_state, +}; + +#[test] +fn test_base() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, Subscription::Tagesschau, 4); + + let first_hash = &util::get_first_hash(&env); + env.run(&["select", "drop", first_hash]); + + let mut map = HashMap::new(); + map.insert("EDITOR", "true"); + env.run_env(&["select", "file", "--done"], &map); + + fs::remove_file(&env.paths.last_selection).unwrap(); + + env.run_env(&["select", "split", "--done"], &map); + assert_states(&env); + + env.run_env(&["select", "file", "--use-last-selection"], &map); + assert_states(&env); +} + +fn assert_states(env: &TestEnv) { + assert_eq!(get_videos_in_state(env, "Picked"), 3); + assert_eq!(get_videos_in_state(env, "Drop"), 1); + + assert_eq!(get_videos_in_state(env, "Watch"), 0); + assert_eq!(get_videos_in_state(env, "Cached"), 0); + assert_eq!(get_videos_in_state(env, "Watched"), 0); +} diff --git a/crates/yt/tests/select/file.rs b/crates/yt/tests/select/file.rs new file mode 100644 index 0000000..b8bd2b5 --- /dev/null +++ b/crates/yt/tests/select/file.rs @@ -0,0 +1,31 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + _testenv::util::{self, Subscription}, + select::get_videos_in_state, + testenv::TestEnv, +}; + +#[test] +fn test_file() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, Subscription::Tagesschau, 4); + + util::run_select(&env, "s/pick/watch/"); + + assert_eq!(get_videos_in_state(&env, "Picked"), 0); + assert_eq!(get_videos_in_state(&env, "Drop"), 0); + + assert_eq!(get_videos_in_state(&env, "Watch"), 4); + assert_eq!(get_videos_in_state(&env, "Cached"), 0); + assert_eq!(get_videos_in_state(&env, "Watched"), 0); +} diff --git a/crates/yt/tests/select/mod.rs b/crates/yt/tests/select/mod.rs new file mode 100644 index 0000000..d7033f8 --- /dev/null +++ b/crates/yt/tests/select/mod.rs @@ -0,0 +1,25 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::_testenv::TestEnv; + +mod base; +mod file; +mod options; + +fn get_videos_in_state(env: &TestEnv, state: &str) -> usize { + let status = env.run(&[ + "status", + "--format", + format!("{{{}_videos_len}}", state.to_lowercase()).as_str(), + ]); + + status.parse().unwrap() +} diff --git a/crates/yt/tests/select/options.rs b/crates/yt/tests/select/options.rs new file mode 100644 index 0000000..6a0d155 --- /dev/null +++ b/crates/yt/tests/select/options.rs @@ -0,0 +1,51 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{_testenv::util, select::get_videos_in_state, testenv::TestEnv}; + +#[test] +fn test_options() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, util::Subscription::Tagesschau, 1); + + let video_hash = &util::get_first_hash(&env); + env.run(&["select", "watch", video_hash]); + + env.assert_output(&["videos", "info", video_hash, "--format", "{options}"], ""); + + env.run(&[ + "select", + "watch", + video_hash, + "--playback-speed", + "1", + "--subtitle-langs", + "en,de,sv", + ]); + + env.assert_output( + &["videos", "info", video_hash, "--format", "{options}"], + "--playback-speed '1' --subtitle-langs 'en,de,sv'", + ); + + env.run(&["select", "watch", video_hash, "-s", "1.7", "-l", "de"]); + + env.assert_output( + &["videos", "info", video_hash, "--format", "{options}"], + "--playback-speed '1.7' --subtitle-langs 'de'", + ); + + assert_eq!(get_videos_in_state(&env, "Picked"), 0); + assert_eq!(get_videos_in_state(&env, "Drop"), 0); + assert_eq!(get_videos_in_state(&env, "Watch"), 1); + assert_eq!(get_videos_in_state(&env, "Cached"), 0); + assert_eq!(get_videos_in_state(&env, "Watched"), 0); +} diff --git a/crates/yt/tests/subscriptions/import_export/golden.txt b/crates/yt/tests/subscriptions/import_export/golden.txt new file mode 100644 index 0000000..7ed5419 --- /dev/null +++ b/crates/yt/tests/subscriptions/import_export/golden.txt @@ -0,0 +1,2 @@ +Nyheter på lätt svenska: 'https://www.svtplay.se/nyheter-pa-latt-svenska' +tagesschau: 'https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU' diff --git a/crates/bytes/Cargo.lock.license b/crates/yt/tests/subscriptions/import_export/golden.txt.license index d4d410f..7813eb6 100644 --- a/crates/bytes/Cargo.lock.license +++ b/crates/yt/tests/subscriptions/import_export/golden.txt.license @@ -1,6 +1,6 @@ yt - A fully featured command line YouTube client -Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> SPDX-License-Identifier: GPL-3.0-or-later This file is part of Yt. diff --git a/crates/yt/tests/subscriptions/import_export/mod.rs b/crates/yt/tests/subscriptions/import_export/mod.rs new file mode 100644 index 0000000..1156508 --- /dev/null +++ b/crates/yt/tests/subscriptions/import_export/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::testenv::TestEnv; + +#[test] +fn test_import_export() { + let env = TestEnv::new(module_path!()); + + env.run(&[ + "subs", + "add", + "https://www.svtplay.se/nyheter-pa-latt-svenska", + ]); + env.run(&[ + "subs", + "add", + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU", + ]); + + let before = env.run(&["subs", "list"]); + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); + + env.run_piped(&["subs", "export"], &["subs", "import", "--force"]); + + env.assert_output(&["subs", "list"], &before); + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); +} diff --git a/crates/yt/tests/subscriptions/mod.rs b/crates/yt/tests/subscriptions/mod.rs new file mode 100644 index 0000000..0b300c5 --- /dev/null +++ b/crates/yt/tests/subscriptions/mod.rs @@ -0,0 +1,12 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +mod import_export; +mod naming_subscriptions; diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt new file mode 100644 index 0000000..46ede50 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt @@ -0,0 +1,2 @@ +Nyheter: 'https://www.svtplay.se/nyheter-pa-latt-svenska' +Vewn: 'https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU' diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license new file mode 100644 index 0000000..7813eb6 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license @@ -0,0 +1,9 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +SPDX-License-Identifier: GPL-3.0-or-later + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs b/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs new file mode 100644 index 0000000..50fe3e4 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs @@ -0,0 +1,33 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::testenv::TestEnv; + +#[test] +fn test_naming_subscriptions() { + let env = TestEnv::new(module_path!()); + + env.run(&[ + "subs", + "add", + "https://www.svtplay.se/nyheter-pa-latt-svenska", + "--name", + "Nyheter", + ]); + env.run(&[ + "subs", + "add", + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU", + "--name", + "Vewn", + ]); + + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); +} diff --git a/crates/yt/tests/tests.rs b/crates/yt/tests/tests.rs new file mode 100644 index 0000000..89c3091 --- /dev/null +++ b/crates/yt/tests/tests.rs @@ -0,0 +1,22 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// Use this, for the background run pids +// #![feature(linux_pidfd)] + +#![allow(unused_crate_dependencies)] + +mod _testenv; +pub(crate) use _testenv as testenv; + +mod select; +mod subscriptions; +mod videos; +mod watch; diff --git a/crates/yt/tests/videos/downloading.rs b/crates/yt/tests/videos/downloading.rs new file mode 100644 index 0000000..f026858 --- /dev/null +++ b/crates/yt/tests/videos/downloading.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{_testenv::util, testenv::TestEnv}; + +#[test] +fn test_downloading() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, util::Subscription::Tagesschau, 1); + + let first_hash = &util::get_first_hash(&env); + env.run(&["select", "watch", first_hash]); + + env.run(&["download"]); + + let usage = get_cache_usage(&env); + assert!(usage > 0.0); + + env.run(&["cache", "clear"]); + + let usage = get_cache_usage(&env); + + #[allow(clippy::float_cmp)] + { + assert_eq!(usage, 0.0); + } +} + +fn get_cache_usage(env: &TestEnv) -> f64 { + let status = env.run(&["status", "--format", "{cache_usage}"]); + + let split: Vec<_> = status.split(' ').collect(); + let usage: f64 = split[0].parse().unwrap(); + let unit = split[1]; + + #[allow(clippy::cast_precision_loss)] + match unit { + "B" => usage * (1024u64.pow(0)) as f64, + "KiB" => usage * (1024u64.pow(1)) as f64, + "MiB" => usage * (1024u64.pow(2)) as f64, + "GiB" => usage * (1024u64.pow(3)) as f64, + other => unreachable!("Unknown unit: {other}"), + } +} diff --git a/crates/yt/tests/videos/mod.rs b/crates/yt/tests/videos/mod.rs new file mode 100644 index 0000000..6a80761 --- /dev/null +++ b/crates/yt/tests/videos/mod.rs @@ -0,0 +1,11 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +mod downloading; diff --git a/crates/yt/tests/watch/focus_switch.rs b/crates/yt/tests/watch/focus_switch.rs new file mode 100644 index 0000000..81246f3 --- /dev/null +++ b/crates/yt/tests/watch/focus_switch.rs @@ -0,0 +1,53 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use yt_dlp::json_cast; + +use crate::{_testenv::util, testenv::TestEnv, watch::MpvControl}; + +#[test] +#[ignore = "Currently, this test is missing it's goal"] +fn test_focus_switch() { + let mut env = TestEnv::new(module_path!()); + + { + util::provide_videos(&env, util::Subscription::Tagesschau, 32); + + util::run_select(&env, "s/pick/watch/"); + + env.run(&["download"]); + } + + let mut mpv = MpvControl::new(&mut env); + + assert_pos(&mut mpv, 0); + + for i in 1..32 { + mpv.assert(&["playlist-next", "weak"]); + assert_pos(&mut mpv, i); + } + + mpv.assert(&["playlist-next", "weak"]); + assert_pos(&mut mpv, 2); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 1); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 0); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 0); +} + +fn assert_pos(mpv: &mut MpvControl, pos: i64) { + let mpv_pos = mpv.assert(&["get_property", "playlist-pos"]); + assert_eq!(json_cast!(mpv_pos, as_i64), pos); +} diff --git a/crates/yt/tests/watch/mod.rs b/crates/yt/tests/watch/mod.rs new file mode 100644 index 0000000..7af8b39 --- /dev/null +++ b/crates/yt/tests/watch/mod.rs @@ -0,0 +1,135 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + io::{BufRead, BufReader, Write}, + os::unix::net::UnixStream, + path::PathBuf, + sync::atomic::AtomicU64, +}; + +use colors::{Colorize, IntoCanvas}; +use serde_json::json; +use yt_dlp::{json_cast, json_get, json_try_get}; + +use crate::_testenv::TestEnv; + +mod focus_switch; + +struct MpvControl { + stream: UnixStream, + current_request_id: AtomicU64, + name: &'static str, +} + +impl MpvControl { + fn new(env: &mut TestEnv) -> Self { + let socket_path = { + let stdout = env.run_background(&[ + "watch", + // "--headless", + "--provide-ipc-socket", + ]); + + let line = { + let mut buf = String::new(); + let mut reader = BufReader::new(stdout); + reader.read_line(&mut buf).expect("In-memory"); + buf + }; + + PathBuf::from(line.trim()) + }; + + let stream = UnixStream::connect(&socket_path).unwrap_or_else(|e| { + panic!( + "Path to socket ('{}') should exist, but did not: {e}", + socket_path.display() + ) + }); + + let mut me = Self { + stream, + name: env.name, + current_request_id: AtomicU64::new(0), + }; + + // Disable all events. + // We do not use them, and this should reduce the read load on the socket. + me.assert(&["disable_event", "all"]); + + me + } + + /// Run a command and assert that it ran successfully. + fn assert(&mut self, args: &[&str]) -> serde_json::Value { + let out = self.command(args); + + out.unwrap_or_else(|e| panic!("`mpv {}` failed; error {e}.", args.join(" "))) + } + + /// Run a command in mpv. + /// Will return true if the command ran correctly and false if not. + fn command(&mut self, args: &[&str]) -> Result<serde_json::Value, String> { + let tl_rid = self + .current_request_id + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + eprint!( + "{} `mpv {}`", + self.name.blue().italic().render(true), + args.join(" ") + ); + + writeln!( + self.stream, + "{}", + json!( { "command": args, "request_id": tl_rid }) + ) + .expect("Should always work"); + + loop { + let response: serde_json::Value = { + let mut reader = BufReader::new(&mut self.stream); + + let mut buf = String::new(); + reader.read_line(&mut buf).expect("Works"); + serde_json::from_str(&buf).expect("Mpv only returns json") + }; + + if let Some(rid) = json_try_get!(response, "request_id", as_u64) { + if rid == tl_rid { + let error = json_get!(response, "error", as_str); + + if error == "success" { + let data: serde_json::Value = { + match response.get("data") { + Some(val) => val.to_owned(), + None => serde_json::Value::Null, + } + }; + + eprintln!(", {}: {data}", "output".bright_blue().render(true),); + return Ok(data); + } + + eprintln!(", {}: {error}", "error".bright_red().render(true)); + return Err(error.to_owned()); + } + } + } + } +} + +impl Drop for MpvControl { + fn drop(&mut self) { + self.assert(&["quit"]); + } +} diff --git a/crates/yt_dlp/Cargo.toml b/crates/yt_dlp/Cargo.toml index 90f2e10..eb2924d 100644 --- a/crates/yt_dlp/Cargo.toml +++ b/crates/yt_dlp/Cargo.toml @@ -22,11 +22,13 @@ rust-version.workspace = true publish = true [dependencies] -indexmap = { version = "2.9.0", default-features = false } +curl = "0.4.49" log.workspace = true -rustpython = { git = "https://github.com/RustPython/RustPython.git", features = ["threading", "stdlib", "stdio", "importlib", "ssl"], default-features = false } +pyo3 = { workspace = true } +pyo3-pylogger = { path = "crates/pyo3-pylogger" } +serde = { workspace = true, features = ["derive"] } serde_json.workspace = true -thiserror = "2.0.12" +thiserror = "2.0.17" url.workspace = true [lints] diff --git a/crates/yt_dlp/README.md b/crates/yt_dlp/README.md index 591ef2e..ece8540 100644 --- a/crates/yt_dlp/README.md +++ b/crates/yt_dlp/README.md @@ -12,7 +12,7 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. # Yt_py -> \[can be empty\] +> [can be empty] Some text about the project. diff --git a/crates/yt_dlp/crates/pyo3-pylogger/.gitignore b/crates/yt_dlp/crates/pyo3-pylogger/.gitignore new file mode 100644 index 0000000..733c5bc --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/.gitignore @@ -0,0 +1,13 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +target +Cargo.lock +.idea diff --git a/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml b/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml new file mode 100644 index 0000000..a2676e7 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml @@ -0,0 +1,31 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +[package] +name = "pyo3-pylogger" +version = "1.9.0" +edition = "2021" +authors = [ + "Dylan Bobby Storey <dylan.storey@gmail.com>", + "cpu <daniel@binaryparadox.net>", + "Warren Snipes <contact@warrensnipes.dev>", +] +description = "Enables `log` for pyo3 based Rust applications using the `logging` modules." +publish = ["crates-io"] +license = "Apache-2.0" +readme = "README.md" +homepage = "https://github.com/dylanbstorey/pyo3-pylogger" +repository = "https://github.com/dylanbstorey/pyo3-pylogger" +documentation = "https://github.com/dylanbstorey/pyo3-pylogger" + +[dependencies] +pyo3 = { workspace = true } +log = { workspace = true } +phf = { version = "0.13", features = ["macros"] } diff --git a/crates/yt_dlp/crates/pyo3-pylogger/LICENSE b/crates/yt_dlp/crates/pyo3-pylogger/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/yt_dlp/crates/pyo3-pylogger/README.md b/crates/yt_dlp/crates/pyo3-pylogger/README.md new file mode 100644 index 0000000..e68903b --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/README.md @@ -0,0 +1,160 @@ +<!-- +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +SPDX-License-Identifier: Apache-2.0 + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. +--> + +# pyo3-pylogger + +Enables log messages for pyo3 embedded Python applications using Python's +`logging` or module. + +# Features + +- Logging integration between Python's `logging` module and Rust's `log` crate +- Structured logging support via the logging + [extra](https://docs.python.org/3/library/logging.html#logging.Logger.debug) + field (requires `kv` or `tracing-kv`feature) +- Integration with Rust's `tracing` library (requires `tracing` feature) + +# Usage + +```rust +use log::{info, warn}; +use pyo3::{ffi::c_str, prelude::*}; +fn main() { + // register the host handler with python logger, providing a logger target + pyo3_pylogger::register("example_application_py_logger"); + + // initialize up a logger + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace")).init(); + //just show the logger working from Rust. + info!("Just some normal information!"); + warn!("Something spooky happened!"); + + // Ask pyo3 to set up embedded Python interpreter + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + // Python code can now `import logging` as usual + py.run( + c_str!( + r#" +import logging +logging.getLogger().setLevel(0) +logging.debug('DEBUG') +logging.info('INFO') +logging.warning('WARNING') +logging.error('ERROR') +logging.getLogger('foo.bar.baz').info('INFO')"# + ), + None, + None, + ) + .unwrap(); + }) +} + + +``` + +## Outputs + +```bash +[2025-03-28T01:12:29Z INFO helloworld] Just some normal information! +[2025-03-28T01:12:29Z WARN helloworld] Something spooky happened! +[2025-03-28T01:12:29Z DEBUG example_application_py_logger] DEBUG +[2025-03-28T01:12:29Z INFO example_application_py_logger] INFO +[2025-03-28T01:12:29Z WARN example_application_py_logger] WARNING +[2025-03-28T01:12:29Z ERROR example_application_py_logger] ERROR +[2025-03-28T01:12:29Z INFO example_application_py_logger::foo::bar::baz] INFO +``` + +## Structured Logging + +To enable structured logging support, add the `kv` feature to your `Cargo.toml`: + +```toml +[dependencies] +pyo3-pylogger = { version = "0.4", features = ["kv"] } +``` + +Then you can use Python's `extra` parameter to pass structured data: + +```python +logging.info("Processing order", extra={"order_id": "12345", "amount": 99.99}) +``` + +When using a structured logging subscriber in Rust, these key-value pairs will +be properly captured, for example: + +```bash +[2025-03-28T01:12:29Z INFO example_application_py_logger] Processing order order_id=12345 amount=99.99 +``` + +## Tracing Support + +To enable integration with Rust's `tracing` library, add the `tracing` feature +to your `Cargo.toml`: + +```toml +[dependencies] +pyo3-pylogger = { version = "0.4", default-features = false, features = ["tracing"] } +``` + +When the `tracing` feature is enabled, Python logs will be forwarded to the +active tracing subscriber: + +```rust +use tracing::{info, warn}; +use pyo3::{ffi::c_str, prelude::*}; + +fn main() { + // Register the tracing handler with Python logger + pyo3_pylogger::register_tracing("example_application_py_logger"); + + // Initialize tracing subscriber + tracing_subscriber::fmt::init(); + + // Tracing events from Rust + info!("Tracing information from Rust"); + + // Python logging will be captured by the tracing subscriber + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + py.run( + c_str!( + r#" +import logging +logging.getLogger().setLevel(0) +logging.info('This will be captured by tracing')"# + ), + None, + None, + ) + .unwrap(); + }) +} +``` + +### Structured Data with Tracing + +The `tracing` feature automatically supports Python's `extra` field for +structured data. However, the KV fields are json serialized and not available as +tracing attributes. This is a limitation of the `tracing` library and is not +specific to this crate. See +[this issue](https://github.com/tokio-rs/tracing/issues/372) for more +information. + +# Feature Flags + +- `kv`: Enables structured logging support via Python's `extra` fields. This + adds support for the `log` crate's key-value system. +- `tracing`: Enables integration with Rust's `tracing` library. +- `tracing-kv`: Enables structured logging support via Python's `extra` fields + and integration with Rust's `tracing` library. diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs new file mode 100644 index 0000000..67a0c3e --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs @@ -0,0 +1,127 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! Key-Value handling module for Python LogRecord attributes. +//! +//! This module provides functionality to extract and handle custom key-value pairs +//! from Python LogRecord objects, facilitating integration between Python's logging +//! system and Rust's log crate. + +use pyo3::{ + Bound, PyAny, PyResult, + types::{PyAnyMethods, PyDict, PyDictMethods, PyListMethods}, +}; +use std::collections::HashMap; + +/// A static hashset containing all standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes defined in the CPython logging module. +/// +/// This set is used to differentiate between standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes and custom key-value pairs +/// that users might add to their log records. The attributes listed here correspond to the default +/// attributes created by Python's [makeRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L1633-L1634) function. +pub static LOG_RECORD_KV_ATTRIBUTES: phf::Set<&'static str> = phf::phf_set! { + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "taskName", +}; + +/// Extracts custom key-value pairs from a Python LogRecord object. +/// +/// This function examines the `__dict__` of a LogRecord(https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) object and identifies any attributes +/// that are not part of the standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes. These custom attributes are +/// treated as key-value pairs for structured logging. +/// +/// # Arguments +/// * `record` - A reference to a Python LogRecord object +/// +/// # Returns +/// * `PyResult<Option<HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>>>` - If custom attributes +/// are found, returns a HashMap containing the key-value pairs. Returns None if no custom +/// attributes are present. +/// +/// # Note +/// This function relies on the fact that Python will not implement new attributes on the LogRecord object. +/// If new attributes are added, this function will not be able to filter them out and will return them as key-value pairs. +/// In that future, [LOG_RECORD_KV_ATTRIBUTES] will need to be updated to include the new attributes. +/// This is an unfortunate side effect of using the `__dict__` attribute to extract key-value pairs. However, there are no other ways to handle this given that CPython does not distinguish between user-provided attributes and attributes created by the logging module. +pub fn find_kv_args<'a>( + record: &Bound<'a, PyAny>, +) -> PyResult<Option<std::collections::HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>>> { + let dict: Bound<'_, PyDict> = record.getattr("__dict__")?.extract()?; + + // We can abuse the fact that Python dictionaries are ordered by insertion order to reverse iterate over the keys + // and stop at the first key that is not a predefined key-value pair attribute. + let mut kv_args: Option<HashMap<String, pyo3::Bound<'_, pyo3::PyAny>>> = None; + + for item in dict.items().iter().rev() { + let (key, value) = + item.extract::<(pyo3::Bound<'_, pyo3::PyAny>, pyo3::Bound<'_, pyo3::PyAny>)>()?; + + let key_str = key.to_string(); + if LOG_RECORD_KV_ATTRIBUTES.contains(&key_str) { + break; + } + if kv_args.is_none() { + kv_args = Some(HashMap::new()); + } + + kv_args.as_mut().unwrap().insert(key_str, value); + } + + Ok(kv_args) +} + +/// A wrapper struct that implements the `log::kv::Source` trait for Python key-value pairs. +/// +/// This struct allows Python LogRecord custom attributes to be used with Rust's +/// structured logging system by implementing the necessary trait for key-value handling. +/// +/// # Type Parameters +/// * `'a` - The lifetime of the contained Python values +pub struct KVSource<'a>(pub HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>); + +impl log::kv::Source for KVSource<'_> { + /// Visits each key-value pair in the source, converting Python values to debug representations. + /// + /// # Arguments + /// * `visitor` - The visitor that will process each key-value pair + /// + /// # Returns + /// * `Result<(), log::kv::Error>` - Success if all pairs are visited successfully, + /// or an error if visitation fails + fn visit<'kvs>( + &'kvs self, + visitor: &mut dyn log::kv::VisitSource<'kvs>, + ) -> Result<(), log::kv::Error> { + for (key, value) in &self.0 { + let v: log::kv::Value<'_> = log::kv::Value::from_debug(value); + + visitor.visit_pair(log::kv::Key::from_str(key), v)?; + } + Ok(()) + } +} diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs new file mode 100644 index 0000000..d244ef4 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs @@ -0,0 +1,43 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +/// A wrapper type for logging levels that supports both `tracing` and `log` features. +pub(crate) struct Level(pub log::Level); + +/// Converts a numeric level value to the appropriate logging Level. +/// +/// # Arguments +/// +/// * `level` - A u8 value representing the logging level: +/// * 40+ = Error +/// * 30-39 = Warn +/// * 20-29 = Info +/// * 10-19 = Debug +/// * 0-9 = Trace +/// +/// # Returns +/// +/// Returns a `Level` wrapper containing either a `tracing::Level` or `log::Level` +/// depending on which feature is enabled. +pub(crate) fn get_level(level: u8) -> Level { + { + if level.ge(&40u8) { + Level(log::Level::Error) + } else if level.ge(&30u8) { + Level(log::Level::Warn) + } else if level.ge(&20u8) { + Level(log::Level::Info) + } else if level.ge(&10u8) { + Level(log::Level::Debug) + } else { + Level(log::Level::Trace) + } + } +} diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs new file mode 100644 index 0000000..3ecb123 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs @@ -0,0 +1,211 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + ffi::CString, + sync::{self, OnceLock}, +}; + +use log::{debug, log_enabled}; +use pyo3::{ + Bound, Py, PyAny, PyResult, Python, pyfunction, + sync::OnceLockExt, + types::{PyAnyMethods, PyDict, PyListMethods, PyModuleMethods}, + wrap_pyfunction, +}; + +mod kv; +mod level; + +static LOGGER: sync::OnceLock<Py<PyAny>> = OnceLock::new(); + +/// Is the specified record to be logged? Returns false for no, +/// true for yes. Filters can either modify log records in-place or +/// return a completely different record instance which will replace +/// the original log record in any future processing of the event. +#[pyfunction] +fn filter_error_log<'py>(record: Bound<'py, PyAny>) -> bool { + // Filter out all error logs (they are propagated as rust errors) + let levelname: String = record + .getattr("levelname") + .expect("This should exist") + .extract() + .expect("This should be a String"); + + let return_value = levelname.as_str() != "ERROR"; + + if log_enabled!(log::Level::Debug) && !return_value { + let message: String = { + let get_message = record.getattr("getMessage").expect("Is set"); + let message: String = get_message + .call((), None) + .expect("Can be called") + .extract() + .expect("Downcasting works"); + + message.as_str().to_owned() + }; + + debug!("Swollowed error message: '{message}'"); + } + return_value +} + +/// Consume a Python `logging.LogRecord` and emit a Rust `Log` instead. +#[pyfunction] +fn host_log(record: Bound<'_, PyAny>, rust_target: &str) -> PyResult<()> { + let level = record.getattr("levelno")?.extract()?; + let message = record.getattr("getMessage")?.call0()?.to_string(); + let pathname = record.getattr("pathname")?.extract::<String>()?; + let lineno = record.getattr("lineno")?.extract::<u32>()?; + + let logger_name = record.getattr("name")?.extract::<String>()?; + + let full_target: Option<String> = if logger_name.trim().is_empty() || logger_name == "root" { + None + } else { + // Libraries (ex: tracing_subscriber::filter::Directive) expect rust-style targets like foo::bar, + // and may not deal well with "." as a module separator: + let logger_name = logger_name.replace('.', "::"); + Some(format!("{rust_target}::{logger_name}")) + }; + let target = full_target.as_deref().unwrap_or(rust_target); + + handle_record(record, target, &message, lineno, &pathname, level)?; + + Ok(()) +} + +fn handle_record( + #[allow(unused_variables)] record: Bound<'_, PyAny>, + target: &str, + message: &str, + lineno: u32, + pathname: &str, + level: u8, +) -> PyResult<()> { + // If log feature is enabled, use log::logger + let level = crate::level::get_level(level).0; + + { + let mut metadata_builder = log::MetadataBuilder::new(); + metadata_builder.target(target); + metadata_builder.level(level); + + let mut record_builder = log::Record::builder(); + + { + let kv_args = kv::find_kv_args(&record)?; + + let kv_source = kv_args.map(kv::KVSource); + if let Some(kv_source) = kv_source { + log::logger().log( + &record_builder + .metadata(metadata_builder.build()) + .args(format_args!("{}", &message)) + .line(Some(lineno)) + .file(Some(pathname)) + .module_path(Some(pathname)) + .key_values(&kv_source) + .build(), + ); + return Ok(()); + } + } + + log::logger().log( + &record_builder + .metadata(metadata_builder.build()) + .args(format_args!("{}", &message)) + .line(Some(lineno)) + .file(Some(pathname)) + .module_path(Some(pathname)) + .build(), + ); + } + + Ok(()) +} + +/// Registers the host_log function in rust as the event handler for Python's logging logger +/// This function needs to be called from within a pyo3 context as early as possible to ensure logging messages +/// arrive to the rust consumer. +pub fn setup_logging<'py>(py: Python<'py>, target: &str) -> PyResult<Bound<'py, PyAny>> { + let logger = LOGGER + .get_or_init_py_attached(py, || match setup_logging_inner(py, target) { + Ok(ok) => ok.unbind(), + Err(err) => { + panic!("Failed to initialize logger: {}", err); + } + }) + .clone_ref(py); + + Ok(logger.into_bound(py)) +} + +fn setup_logging_inner<'py>(py: Python<'py>, target: &str) -> PyResult<Bound<'py, PyAny>> { + let logging = py.import("logging")?; + + logging.setattr("host_log", wrap_pyfunction!(host_log, &logging)?)?; + + #[allow(clippy::uninlined_format_args)] + let code = CString::new(format!( + r#" +class HostHandler(Handler): + def __init__(self, level=0): + super().__init__(level=level) + + def emit(self, record: LogRecord): + host_log(record, "{}") + +oldBasicConfig = basicConfig +def basicConfig(*pargs, **kwargs): + if "handlers" not in kwargs: + kwargs["handlers"] = [HostHandler()] + return oldBasicConfig(*pargs, **kwargs) +"#, + target + ))?; + + let logging_scope = logging.dict(); + py.run(&code, Some(&logging_scope), None)?; + + let all = logging.index()?; + all.append("HostHandler")?; + + let logger = { + let get_logger = logging_scope.get_item("getLogger")?; + get_logger.call((target,), None)? + }; + + { + let basic_config = logging_scope.get_item("basicConfig")?; + basic_config.call( + (), + { + let dict = PyDict::new(py); + + // Ensure that all events are logged by setting + // the log level to NOTSET (we filter on rust's side) + dict.set_item("level", 0)?; + + Some(dict) + } + .as_ref(), + )?; + } + + { + let add_filter = logger.getattr("addFilter")?; + add_filter.call((wrap_pyfunction!(filter_error_log, &logging)?,), None)?; + } + + Ok(logger) +} diff --git a/crates/bytes/update.sh b/crates/yt_dlp/crates/pyo3-pylogger/update.sh index c1a0215..dd3e57e 100755 --- a/crates/bytes/update.sh +++ b/crates/yt_dlp/crates/pyo3-pylogger/update.sh @@ -1,9 +1,9 @@ -#!/usr/bin/env sh +#! /usr/bin/env sh # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 # # This file is part of Yt. # @@ -13,3 +13,5 @@ cd "$(dirname "$0")" || exit 1 [ "$1" = "upgrade" ] && cargo upgrade --incompatible cargo update + +# vim: ft=sh diff --git a/crates/yt_dlp/examples/main.rs b/crates/yt_dlp/examples/main.rs new file mode 100644 index 0000000..e924407 --- /dev/null +++ b/crates/yt_dlp/examples/main.rs @@ -0,0 +1,15 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +fn main() { + let yt_dlp = yt_dlp::options::YoutubeDLOptions::new().build().unwrap(); + + dbg!(yt_dlp.version().unwrap()); +} diff --git a/crates/yt_dlp/src/info_json.rs b/crates/yt_dlp/src/info_json.rs new file mode 100644 index 0000000..402acb4 --- /dev/null +++ b/crates/yt_dlp/src/info_json.rs @@ -0,0 +1,56 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use pyo3::{ + Bound, Python, intern, + types::{PyAnyMethods, PyDict}, +}; + +pub type InfoJson = serde_json::Map<String, serde_json::Value>; + +/// # Panics +/// If expectation about python operations fail. +#[must_use] +pub fn json_loads( + input: serde_json::Map<String, serde_json::Value>, + py: Python<'_>, +) -> Bound<'_, PyDict> { + let json = py.import(intern!(py, "json")).expect("Module exists"); + let loads = json.getattr(intern!(py, "loads")).expect("Method exists"); + let self_str = serde_json::to_string(&serde_json::Value::Object(input)).expect("Vaild json"); + let dict = loads + .call((self_str,), None) + .expect("Vaild json is always a valid dict"); + + dict.cast_into().expect("Should always be a dict") +} + +/// # Panics +/// If expectation about python operations fail. +#[must_use] +pub fn json_dumps(input: &Bound<'_, PyDict>) -> serde_json::Map<String, serde_json::Value> { + let py = input.py(); + + let json = py.import(intern!(py, "json")).expect("Module exists"); + let dumps = json.getattr(intern!(py, "dumps")).expect("Method exists"); + let dict = dumps + .call((input,), None) + .map_err(|err| err.print(py)) + .expect("Might not always work, but for our dicts it works"); + + let string: String = dict.extract().expect("Should always be a string"); + + let value: serde_json::Value = serde_json::from_str(&string).expect("Should be valid json"); + + match value { + serde_json::Value::Object(map) => map, + _ => unreachable!("These should not be json.dumps output"), + } +} diff --git a/crates/yt_dlp/src/lib.rs b/crates/yt_dlp/src/lib.rs index dd42fc6..4b252de 100644 --- a/crates/yt_dlp/src/lib.rs +++ b/crates/yt_dlp/src/lib.rs @@ -1,216 +1,137 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + //! The `yt_dlp` interface is completely contained in the [`YoutubeDL`] structure. -use std::{self, env, mem, path::PathBuf}; - -use indexmap::IndexMap; -use log::{Level, debug, error, info, log_enabled}; -use logging::setup_logging; -use rustpython::{ - InterpreterConfig, - vm::{ - self, AsObject, Interpreter, PyObjectRef, PyPayload, PyRef, VirtualMachine, - builtins::{PyBaseException, PyBaseExceptionRef, PyDict, PyList, PyStr}, - function::{FuncArgs, KwArgs, PosArgs}, - py_io::Write, - suggestion::offer_suggestions, - }, +use std::path::PathBuf; + +use log::{debug, info}; +use pyo3::{ + Bound, Py, PyAny, Python, intern, + types::{PyAnyMethods, PyDict, PyIterator, PyList}, }; use url::Url; -mod logging; +use crate::{ + info_json::{InfoJson, json_dumps, json_loads}, + python_error::{IntoPythonError, PythonError}, +}; + +pub mod info_json; +pub mod options; +pub mod post_processors; pub mod progress_hook; +pub mod python_error; #[macro_export] macro_rules! json_get { - ($value:expr, $name:literal, $into:ident) => { - $crate::json_cast!($value.get($name).expect("Should exist"), $into) - }; + ($value:expr, $name:literal, $into:ident) => {{ + match $value.get($name) { + Some(val) => $crate::json_cast!(@log_key $name, val, $into), + None => panic!( + concat!( + "Expected '", + $name, + "' to be a key for the '", + stringify!($value), + "' object: {:#?}" + ), + $value + ), + } + }}; +} + +#[macro_export] +macro_rules! json_try_get { + ($value:expr, $name:literal, $into:ident) => {{ + if let Some(val) = $value.get($name) { + if val.is_null() { + None + } else { + Some(json_cast!(@log_key $name, val, $into)) + } + } else { + None + } + }}; } #[macro_export] macro_rules! json_cast { - ($value:expr, $into:ident) => { - $value.$into().expect(concat!( - "Should be able to cast value into ", - stringify!($into) - )) - }; + ($value:expr, $into:ident) => {{ + let value_name = stringify!($value); + json_cast!(@log_key value_name, $value, $into) + }}; + + (@log_key $name:expr, $value:expr, $into:ident) => {{ + match $value.$into() { + Some(result) => result, + None => panic!( + concat!( + "Expected to be able to cast '{}' value (which is '{:?}') ", + stringify!($into) + ), + $name, + $value + ), + } + }}; } +macro_rules! py_kw_args { + ($py:expr => $($kw_arg_name:ident = $kw_arg_val:expr),*) => {{ + use $crate::python_error::IntoPythonError; + + let dict = PyDict::new($py); + + $( + dict.set_item(stringify!($kw_arg_name), $kw_arg_val).wrap_exc($py)?; + )* + + Some(dict) + } + .as_ref()}; +} +pub(crate) use py_kw_args; + /// The core of the `yt_dlp` interface. +#[derive(Debug)] pub struct YoutubeDL { - interpreter: Interpreter, - youtube_dl_class: PyObjectRef, - yt_dlp_module: PyObjectRef, + inner: Py<PyAny>, options: serde_json::Map<String, serde_json::Value>, } -impl std::fmt::Debug for YoutubeDL { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // TODO(@bpeetz): Use something useful here. <2025-06-13> - f.write_str("YoutubeDL") - } -} - impl YoutubeDL { - /// Construct this instance from options. - /// - /// # Panics - /// If `yt_dlp` changed their interface. + /// Fetch the underlying `yt_dlp` and `python` version. /// /// # Errors - /// If a python call fails. - pub fn from_options(mut options: YoutubeDLOptions) -> Result<Self, build::Error> { - let mut settings = vm::Settings::default(); - if let Ok(python_path) = env::var("PYTHONPATH") { - for path in python_path.split(':') { - settings.path_list.push(path.to_owned()); - } - } else { - error!( - "No PYTHONPATH found or invalid utf8. \ - This means, that you probably did not \ - supply the yt_dlp!" - ); - } - - settings.install_signal_handlers = false; - - // NOTE(@bpeetz): Another value leads to an internal codegen error. <2025-06-13> - settings.optimize = 0; - - settings.isolated = true; - - let interpreter = InterpreterConfig::new() - .init_stdlib() - .settings(settings) - .interpreter(); - - let output_options = options.options.clone(); - - let (yt_dlp_module, youtube_dl_class) = match interpreter.enter(|vm| { - let yt_dlp_module = vm.import("yt_dlp", 0)?; - let class = yt_dlp_module.get_attr("YoutubeDL", vm)?; - - let maybe_hook = mem::take(&mut options.progress_hook); - let opts = options.into_py_dict(vm); - if let Some(function) = maybe_hook { - opts.get_or_insert(vm, vm.new_pyobj("progress_hooks"), || { - let hook: PyObjectRef = vm.new_function("progress_hook", function).into(); - vm.new_pyobj(vec![hook]) - }) - .expect("Should work?"); - } - - { - // Unconditionally set a logger. - // Otherwise, yt_dlp will log to stderr. - - /// Is the specified record to be logged? Returns false for no, - /// true for yes. Filters can either modify log records in-place or - /// return a completely different record instance which will replace - /// the original log record in any future processing of the event. - fn filter_error_log(mut input: FuncArgs, vm: &VirtualMachine) -> bool { - let record = input.args.remove(0); - - // Filter out all error logs (they are propagated as rust errors) - let levelname: PyRef<PyStr> = record - .get_attr("levelname", vm) - .expect("This should exist") - .downcast() - .expect("This should be a String"); - - let return_value = levelname.as_str() != "ERROR"; - - if log_enabled!(Level::Debug) && !return_value { - let message: String = { - let get_message = record.get_attr("getMessage", vm).expect("Is set"); - let message: PyRef<PyStr> = get_message - .call((), vm) - .expect("Can be called") - .downcast() - .expect("Downcasting works"); - - message.as_str().to_owned() - }; - - debug!("Swollowed error message: '{message}'"); - } - return_value - } - - let logging = setup_logging(vm, "yt_dlp")?; - let ytdl_logger = { - let get_logger = logging.get_item("getLogger", vm)?; - get_logger.call(("yt_dlp",), vm)? - }; - - { - let args = FuncArgs::new( - PosArgs::new(vec![]), - KwArgs::new({ - let mut map = IndexMap::new(); - // Ensure that all events are logged by setting - // the log level to NOTSET (we filter on rust's side) - map.insert("level".to_owned(), vm.new_pyobj(0)); - map - }), - ); - - let basic_config = logging.get_item("basicConfig", vm)?; - basic_config.call(args, vm)?; - } - - { - let add_filter = ytdl_logger.get_attr("addFilter", vm)?; - add_filter.call( - (vm.new_function("yt_dlp_error_filter", filter_error_log),), - vm, - )?; - } - - opts.set_item("logger", ytdl_logger, vm)?; - } - - let youtube_dl_class = class.call((opts,), vm)?; - - Ok::<_, PyRef<PyBaseException>>((yt_dlp_module, youtube_dl_class)) - }) { - Ok(ok) => Ok(ok), - Err(err) => { - // TODO(@bpeetz): Do we want to run `interpreter.finalize` here? <2025-06-14> - // interpreter.finalize(Some(err)); - interpreter.enter(|vm| { - let buffer = process_exception(vm, &err); - Err(build::Error::Python(buffer)) - }) - } - }?; - - Ok(Self { - interpreter, - youtube_dl_class, - yt_dlp_module, - options: output_options, + /// If python attribute access fails. + pub fn version(&self) -> Result<(String, String), PythonError> { + Python::attach(|py| { + let yt_dlp = py + .import(intern!(py, "yt_dlp")) + .wrap_exc(py)? + .getattr(intern!(py, "version")) + .wrap_exc(py)? + .getattr(intern!(py, "__version__")) + .wrap_exc(py)? + .extract() + .wrap_exc(py)?; + + let python = py.version(); + + Ok((yt_dlp, python.to_owned())) }) } - /// # Panics - /// - /// If `yt_dlp` changed their location or type of `__version__`. - pub fn version(&self) -> String { - let str_ref: PyRef<PyStr> = self.interpreter.enter_and_expect( - |vm| { - let version_module = self.yt_dlp_module.get_attr("version", vm)?; - let version = version_module.get_attr("__version__", vm)?; - let version = version.downcast().expect("This should always be a string"); - Ok(version) - }, - "yt_dlp version location has changed", - ); - str_ref.to_string() - } - /// Download a given list of URLs. /// Returns the paths they were downloaded to. /// @@ -224,8 +145,9 @@ impl YoutubeDL { let info_json = self.extract_info(url, true, true)?; // Try to work around yt-dlp type weirdness - let result_string = if let Some(filename) = info_json.get("filename") { - PathBuf::from(json_cast!(filename, as_str)) + let result_string = if let Some(filename) = json_try_get!(info_json, "filename", as_str) + { + PathBuf::from(filename) } else { PathBuf::from(json_get!( json_cast!( @@ -267,63 +189,66 @@ impl YoutubeDL { download: bool, process: bool, ) -> Result<InfoJson, extract_info::Error> { - match self.interpreter.enter(|vm| { - let pos_args = PosArgs::new(vec![vm.new_pyobj(url.to_string())]); - - let kw_args = KwArgs::new({ - let mut map = IndexMap::new(); - map.insert("download".to_owned(), vm.new_pyobj(download)); - map.insert("process".to_owned(), vm.new_pyobj(process)); - map - }); + Python::attach(|py| { + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "extract_info")) + .wrap_exc(py)?; - let fun_args = FuncArgs::new(pos_args, kw_args); - - let inner = self.youtube_dl_class.get_attr("extract_info", vm)?; let result = inner - .call_with_args(fun_args, vm)? - .downcast::<PyDict>() + .call( + (url.to_string(),), + py_kw_args!(py => download = download, process = process), + ) + .wrap_exc(py)? + .cast_into::<PyDict>() .expect("This is a dict"); // Resolve the generator object - if let Ok(generator) = result.get_item("entries", vm) { - if generator.payload_is::<PyList>() { + if let Ok(generator) = result.get_item(intern!(py, "entries")) { + if generator.is_instance_of::<PyList>() { // already resolved. Do nothing - } else { - let max_backlog = self.options.get("playlistend").map_or(10000, |value| { - usize::try_from(value.as_u64().expect("Works")).expect("Should work") - }); + } else if let Ok(generator) = generator.cast::<PyIterator>() { + // A python generator object. + let max_backlog = json_try_get!(self.options, "playlistend", as_u64) + .map_or(10000, |playlistend| { + usize::try_from(playlistend).expect("Should work") + }); let mut out = vec![]; - let next = generator.get_attr("__next__", vm)?; - while let Ok(output) = next.call((), vm) { - out.push(output); + for output in generator { + out.push(output.wrap_exc(py)?); if out.len() == max_backlog { break; } } - result.set_item("entries", vm.new_pyobj(out), vm)?; - } - } - let result = { - let sanitize = self.youtube_dl_class.get_attr("sanitize_info", vm)?; - let value = sanitize.call((result,), vm)?; + result.set_item(intern!(py, "entries"), out).wrap_exc(py)?; + } else { + // Probably some sort of paged list (`OnDemand` or otherwise) + let max_backlog = json_try_get!(self.options, "playlistend", as_u64) + .map_or(10000, |playlistend| { + usize::try_from(playlistend).expect("Should work") + }); - value.downcast::<PyDict>().expect("This should stay a dict") - }; + let next = generator.getattr(intern!(py, "getslice")).wrap_exc(py)?; - let result_json = json_dumps(result, vm); + let output = next + .call((), py_kw_args!(py => start = 0, end = max_backlog)) + .wrap_exc(py)?; - Ok::<_, PyRef<PyBaseException>>(result_json) - }) { - Ok(ok) => Ok(ok), - Err(err) => self.interpreter.enter(|vm| { - let buffer = process_exception(vm, &err); - Err(extract_info::Error::Python(buffer)) - }), - } + result + .set_item(intern!(py, "entries"), output) + .wrap_exc(py)?; + } + } + + let result = self.prepare_info_json(&result, py)?; + + Ok(result) + }) } /// Take the (potentially modified) result of the information extractor (i.e., @@ -344,263 +269,110 @@ impl YoutubeDL { ie_result: InfoJson, download: bool, ) -> Result<InfoJson, process_ie_result::Error> { - match self.interpreter.enter(|vm| { - let pos_args = PosArgs::new(vec![vm.new_pyobj(json_loads(ie_result, vm))]); + Python::attach(|py| { + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "process_ie_result")) + .wrap_exc(py)?; - let kw_args = KwArgs::new({ - let mut map = IndexMap::new(); - map.insert("download".to_owned(), vm.new_pyobj(download)); - map - }); - - let fun_args = FuncArgs::new(pos_args, kw_args); - - let inner = self.youtube_dl_class.get_attr("process_ie_result", vm)?; let result = inner - .call_with_args(fun_args, vm)? - .downcast::<PyDict>() + .call( + (json_loads(ie_result, py),), + py_kw_args!(py => download = download), + ) + .wrap_exc(py)? + .cast_into::<PyDict>() .expect("This is a dict"); - let result = { - let sanitize = self.youtube_dl_class.get_attr("sanitize_info", vm)?; - let value = sanitize.call((result,), vm)?; - - value.downcast::<PyDict>().expect("This should stay a dict") - }; - - let result_json = json_dumps(result, vm); - - Ok::<_, PyRef<PyBaseException>>(result_json) - }) { - Ok(ok) => Ok(ok), - Err(err) => self.interpreter.enter(|vm| { - let buffer = process_exception(vm, &err); - Err(process_ie_result::Error::Python(buffer)) - }), - } - } -} + let result = self.prepare_info_json(&result, py)?; -#[allow(missing_docs)] -pub mod process_ie_result { - #[derive(Debug, thiserror::Error)] - pub enum Error { - #[error("Python threw an exception: {0}")] - Python(String), - } -} -#[allow(missing_docs)] -pub mod extract_info { - #[derive(Debug, thiserror::Error)] - pub enum Error { - #[error("Python threw an exception: {0}")] - Python(String), + Ok(result) + }) } -} - -pub type InfoJson = serde_json::Map<String, serde_json::Value>; -pub type ProgressHookFunction = fn(input: FuncArgs, vm: &VirtualMachine); - -/// Options, that are used to customize the download behaviour. -/// -/// In the future, this might get a Builder api. -/// -/// See `help(yt_dlp.YoutubeDL())` from python for a full list of available options. -#[derive(Default, Debug)] -pub struct YoutubeDLOptions { - options: serde_json::Map<String, serde_json::Value>, - progress_hook: Option<ProgressHookFunction>, -} -impl YoutubeDLOptions { - #[must_use] - pub fn new() -> Self { - Self { - options: serde_json::Map::new(), - progress_hook: None, - } - } + /// Close this [`YoutubeDL`] instance, and stop all currently running downloads. + /// + /// # Errors + /// If python operations fail. + pub fn close(&self) -> Result<(), close::Error> { + Python::attach(|py| { + debug!("Closing YoutubeDL."); - #[must_use] - pub fn set(self, key: impl Into<String>, value: impl Into<serde_json::Value>) -> Self { - let mut options = self.options; - options.insert(key.into(), value.into()); + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "close")) + .wrap_exc(py)?; - Self { - options, - progress_hook: self.progress_hook, - } - } + inner.call0().wrap_exc(py)?; - #[must_use] - pub fn with_progress_hook(self, progress_hook: ProgressHookFunction) -> Self { - if let Some(_previous_hook) = self.progress_hook { - todo!() - } else { - Self { - options: self.options, - progress_hook: Some(progress_hook), - } - } + Ok(()) + }) } - /// # Errors - /// If the underlying [`YoutubeDL::from_options`] errors. - pub fn build(self) -> Result<YoutubeDL, build::Error> { - YoutubeDL::from_options(self) - } + fn prepare_info_json<'py>( + &self, + info: &Bound<'py, PyDict>, + py: Python<'py>, + ) -> Result<InfoJson, prepare::Error> { + let sanitize = self + .inner + .bind(py) + .getattr(intern!(py, "sanitize_info")) + .wrap_exc(py)?; - #[must_use] - pub fn from_json_options(options: serde_json::Map<String, serde_json::Value>) -> Self { - Self { - options, - progress_hook: None, - } - } + let value = sanitize.call((info,), None).wrap_exc(py)?; - #[must_use] - pub fn get(&self, key: &str) -> Option<&serde_json::Value> { - self.options.get(key) - } + let result = value.cast::<PyDict>().expect("This should stay a dict"); - fn into_py_dict(self, vm: &VirtualMachine) -> PyRef<PyDict> { - json_loads(self.options, vm) + Ok(json_dumps(result)) } } #[allow(missing_docs)] -pub mod build { +pub mod close { + use crate::python_error::PythonError; + #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("Python threw an exception: {0}")] - Python(String), - - #[error("Io error: {0}")] - Io(#[from] std::io::Error), + #[error(transparent)] + Python(#[from] PythonError), } } +#[allow(missing_docs)] +pub mod process_ie_result { + use crate::{prepare, python_error::PythonError}; -fn json_loads( - input: serde_json::Map<String, serde_json::Value>, - vm: &VirtualMachine, -) -> PyRef<PyDict> { - let json = vm.import("json", 0).expect("Module exists"); - let loads = json.get_attr("loads", vm).expect("Method exists"); - let self_str = serde_json::to_string(&serde_json::Value::Object(input)).expect("Vaild json"); - let dict = loads - .call((self_str,), vm) - .expect("Vaild json is always a valid dict"); - - dict.downcast().expect("Should always be a dict") -} + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), -/// # Panics -/// If expectation about python operations fail. -pub fn json_dumps( - input: PyRef<PyDict>, - vm: &VirtualMachine, -) -> serde_json::Map<String, serde_json::Value> { - let json = vm.import("json", 0).expect("Module exists"); - let dumps = json.get_attr("dumps", vm).expect("Method exists"); - let dict = dumps - .call((input,), vm) - .map_err(|err| vm.print_exception(err)) - .expect("Might not always work, but for our dicts it works"); - - let string: PyRef<PyStr> = dict.downcast().expect("Should always be a string"); - - let real_string = string.to_str().expect("Should be valid utf8"); - - // { - // let mut file = File::create("debug.dump.json").unwrap(); - // write!(file, "{}", real_string).unwrap(); - // } - - let value: serde_json::Value = serde_json::from_str(real_string).expect("Should be valid json"); - - match value { - serde_json::Value::Object(map) => map, - _ => unreachable!("These should not be json.dumps output"), + #[error("Failed to prepare the info json")] + InfoJsonPrepare(#[from] prepare::Error), } } +#[allow(missing_docs)] +pub mod extract_info { + use crate::{prepare, python_error::PythonError}; -// Inlined and changed from `vm.write_exception_inner` -fn write_exception<W: Write>( - vm: &VirtualMachine, - output: &mut W, - exc: &PyBaseExceptionRef, -) -> Result<(), W::Error> { - let varargs = exc.args(); - let args_repr = { - match varargs.len() { - 0 => vec![], - 1 => { - let args0_repr = if true { - varargs[0] - .str(vm) - .unwrap_or_else(|_| PyStr::from("<element str() failed>").into_ref(&vm.ctx)) - } else { - varargs[0].repr(vm).unwrap_or_else(|_| { - PyStr::from("<element repr() failed>").into_ref(&vm.ctx) - }) - }; - vec![args0_repr] - } - _ => varargs - .iter() - .map(|vararg| { - vararg.repr(vm).unwrap_or_else(|_| { - PyStr::from("<element repr() failed>").into_ref(&vm.ctx) - }) - }) - .collect(), - } - }; - - let exc_class = exc.class(); - - if exc_class.fast_issubclass(vm.ctx.exceptions.syntax_error) { - unreachable!( - "A syntax error should never be raised, \ - as yt_dlp should not have them and neither our embedded code" - ); - } + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), - let exc_name = exc_class.name(); - match args_repr.len() { - 0 => write!(output, "{exc_name}"), - 1 => write!(output, "{}: {}", exc_name, args_repr[0]), - _ => write!( - output, - "{}: ({})", - exc_name, - args_repr - .iter() - .map(|val| val.as_str()) - .collect::<Vec<_>>() - .join(", "), - ), - }?; - - match offer_suggestions(exc, vm) { - Some(suggestions) => { - write!(output, ". Did you mean: '{suggestions}'?") - } - None => Ok(()), + #[error("Failed to prepare the info json")] + InfoJsonPrepare(#[from] prepare::Error), } } +#[allow(missing_docs)] +pub mod prepare { + use crate::python_error::PythonError; -fn process_exception(vm: &VirtualMachine, err: &PyBaseExceptionRef) -> String { - let mut buffer = String::new(); - write_exception(vm, &mut buffer, err) - .expect("We are writing into an *in-memory* string, it will always work"); - - if log_enabled!(Level::Debug) { - let mut output = String::new(); - vm.write_exception(&mut output, err) - .expect("We are writing into an *in-memory* string, it will always work"); - debug!("Python threw an exception: {output}"); + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), } - - buffer } diff --git a/crates/yt_dlp/src/logging.rs b/crates/yt_dlp/src/logging.rs deleted file mode 100644 index 5cb4c1d..0000000 --- a/crates/yt_dlp/src/logging.rs +++ /dev/null @@ -1,197 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// This file is taken from: https://github.com/dylanbstorey/pyo3-pylogger/blob/d89e0d6820ebc4f067647e3b74af59dbc4941dd5/src/lib.rs -// It is licensed under the Apache 2.0 License, copyright up to 2024 by Dylan Storey -// It was modified by Benedikt Peetz 2024, 2025 - -use log::{Level, MetadataBuilder, Record, logger}; -use rustpython::vm::{ - PyObjectRef, PyRef, PyResult, VirtualMachine, - builtins::{PyInt, PyList, PyStr}, - convert::ToPyObject, - function::FuncArgs, -}; - -/// Consume a Python `logging.LogRecord` and emit a Rust `Log` instead. -fn host_log(mut input: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let record = input.args.remove(0); - let rust_target = { - let base: PyRef<PyStr> = input.args.remove(0).downcast().expect("Should be a string"); - base.as_str().to_owned() - }; - - let level = { - let level: PyRef<PyInt> = record - .get_attr("levelno", vm)? - .downcast() - .expect("Should always be an int"); - level.as_u32_mask() - }; - let message = { - let get_message = record.get_attr("getMessage", vm)?; - let message: PyRef<PyStr> = get_message - .call((), vm)? - .downcast() - .expect("Downcasting works"); - - message.as_str().to_owned() - }; - - let pathname = { - let pathname: PyRef<PyStr> = record - .get_attr("pathname", vm)? - .downcast() - .expect("Is a string"); - - pathname.as_str().to_owned() - }; - - let lineno = { - let lineno: PyRef<PyInt> = record - .get_attr("lineno", vm)? - .downcast() - .expect("Is a number"); - - lineno.as_u32_mask() - }; - - let logger_name = { - let name: PyRef<PyStr> = record - .get_attr("name", vm)? - .downcast() - .expect("Should be a string"); - name.as_str().to_owned() - }; - - let full_target: Option<String> = if logger_name.trim().is_empty() || logger_name == "root" { - None - } else { - // Libraries (ex: tracing_subscriber::filter::Directive) expect rust-style targets like foo::bar, - // and may not deal well with "." as a module separator: - let logger_name = logger_name.replace('.', "::"); - Some(format!("{rust_target}::{logger_name}")) - }; - - let target = full_target.as_deref().unwrap_or(&rust_target); - - // error - let error_metadata = if level >= 40 { - MetadataBuilder::new() - .target(target) - .level(Level::Error) - .build() - } else if level >= 30 { - MetadataBuilder::new() - .target(target) - .level(Level::Warn) - .build() - } else if level >= 20 { - MetadataBuilder::new() - .target(target) - .level(Level::Info) - .build() - } else if level >= 10 { - MetadataBuilder::new() - .target(target) - .level(Level::Debug) - .build() - } else { - MetadataBuilder::new() - .target(target) - .level(Level::Trace) - .build() - }; - - logger().log( - &Record::builder() - .metadata(error_metadata) - .args(format_args!("{}", &message)) - .line(Some(lineno)) - .file(None) - .module_path(Some(&pathname)) - .build(), - ); - - Ok(()) -} - -/// Registers the `host_log` function in rust as the event handler for Python's logging logger -/// This function needs to be called from within a pyo3 context as early as possible to ensure logging messages -/// arrive to the rust consumer. -/// -/// # Panics -/// Only if internal assertions fail. -#[allow(clippy::module_name_repetitions)] -pub(super) fn setup_logging(vm: &VirtualMachine, target: &str) -> PyResult<PyObjectRef> { - let logging = vm.import("logging", 0)?; - - let scope = vm.new_scope_with_builtins(); - - for (key, value) in logging.dict().expect("Should be a dict") { - let key: PyRef<PyStr> = key.downcast().expect("Is a string"); - - scope.globals.set_item(key.as_str(), value, vm)?; - } - scope - .globals - .set_item("host_log", vm.new_function("host_log", host_log).into(), vm)?; - - let local_scope = scope.clone(); - vm.run_code_string( - local_scope, - format!( - r#" -class HostHandler(Handler): - def __init__(self, level=0): - super().__init__(level=level) - - def emit(self, record): - host_log(record,"{target}") - -oldBasicConfig = basicConfig -def basicConfig(*pargs, **kwargs): - if "handlers" not in kwargs: - kwargs["handlers"] = [HostHandler()] - return oldBasicConfig(*pargs, **kwargs) -"# - ) - .as_str(), - "<embedded logging inintializing code>".to_owned(), - )?; - - let all: PyRef<PyList> = logging - .get_attr("__all__", vm)? - .downcast() - .expect("Is a list"); - all.borrow_vec_mut().push(vm.new_pyobj("HostHandler")); - - // { - // let logging_dict = logging.dict().expect("Exists"); - // - // for (key, val) in scope.globals { - // let key: PyRef<PyStr> = key.downcast().expect("Is a string"); - // - // if !logging_dict.contains_key(key.as_str(), vm) { - // logging_dict.set_item(key.as_str(), val, vm)?; - // } - // } - // - // for (key, val) in scope.locals { - // let key: PyRef<PyStr> = key.downcast().expect("Is a string"); - // - // if !logging_dict.contains_key(key.as_str(), vm) { - // logging_dict.set_item(key.as_str(), val, vm)?; - // } - // } - // } - - Ok(scope.globals.to_pyobject(vm)) -} diff --git a/crates/yt_dlp/src/options.rs b/crates/yt_dlp/src/options.rs new file mode 100644 index 0000000..4b8906e --- /dev/null +++ b/crates/yt_dlp/src/options.rs @@ -0,0 +1,207 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync; + +use pyo3::{ + Bound, IntoPyObjectExt, PyAny, PyResult, Python, intern, + types::{PyAnyMethods, PyCFunction, PyDict, PyTuple}, +}; +use pyo3_pylogger::setup_logging; + +use crate::{ + YoutubeDL, json_loads, post_processors, py_kw_args, + python_error::{IntoPythonError, PythonError}, +}; + +pub type ProgressHookFunction = fn(py: Python<'_>) -> PyResult<Bound<'_, PyCFunction>>; +pub type PostProcessorFunction = fn(py: Python<'_>) -> PyResult<Bound<'_, PyAny>>; + +/// Options, that are used to customize the download behaviour. +/// +/// In the future, this might get a Builder api. +/// +/// See `help(yt_dlp.YoutubeDL())` from python for a full list of available options. +#[derive(Default, Debug)] +pub struct YoutubeDLOptions { + options: serde_json::Map<String, serde_json::Value>, + progress_hook: Option<ProgressHookFunction>, + post_processors: Vec<PostProcessorFunction>, +} + +impl YoutubeDLOptions { + #[must_use] + pub fn new() -> Self { + let me = Self { + options: serde_json::Map::new(), + progress_hook: None, + post_processors: vec![], + }; + + me.with_post_processor(post_processors::dearrow::process) + } + + #[must_use] + pub fn set(self, key: impl Into<String>, value: impl Into<serde_json::Value>) -> Self { + let mut options = self.options; + options.insert(key.into(), value.into()); + + Self { options, ..self } + } + + #[must_use] + pub fn with_progress_hook(self, progress_hook: ProgressHookFunction) -> Self { + if let Some(_previous_hook) = self.progress_hook { + todo!() + } else { + Self { + progress_hook: Some(progress_hook), + ..self + } + } + } + + #[must_use] + pub fn with_post_processor(mut self, pp: PostProcessorFunction) -> Self { + self.post_processors.push(pp); + self + } + + /// # Errors + /// If the underlying [`YoutubeDL::from_options`] errors. + pub fn build(self) -> Result<YoutubeDL, build::Error> { + YoutubeDL::from_options(self) + } + + #[must_use] + pub fn from_json_options(options: serde_json::Map<String, serde_json::Value>) -> Self { + Self { + options, + ..Self::new() + } + } + + #[must_use] + pub fn get(&self, key: &str) -> Option<&serde_json::Value> { + self.options.get(key) + } +} + +impl YoutubeDL { + /// Construct this instance from options. + /// + /// # Panics + /// If `yt_dlp` changed their interface. + /// + /// # Errors + /// If a python call fails. + #[allow(clippy::too_many_lines)] + pub fn from_options(options: YoutubeDLOptions) -> Result<Self, build::Error> { + Python::initialize(); + + let output_options = options.options.clone(); + + let yt_dlp_module = Python::attach(|py| { + let opts = json_loads(options.options, py); + + { + static CALL_ONCE: sync::Once = sync::Once::new(); + + CALL_ONCE.call_once(|| { + py.run( + c" +import signal +signal.signal(signal.SIGINT, signal.SIG_DFL) + ", + None, + None, + ) + .unwrap_or_else(|err| { + panic!("Failed to disable python signal handling: {err}") + }); + }); + } + + { + // Setup the progress hook + if let Some(ph) = options.progress_hook { + opts.set_item(intern!(py, "progress_hooks"), vec![ph(py).wrap_exc(py)?]) + .wrap_exc(py)?; + } + } + + { + // Unconditionally set a logger. + // Otherwise, yt_dlp will log to stderr. + + let ytdl_logger = setup_logging(py, "yt_dlp").wrap_exc(py)?; + + opts.set_item(intern!(py, "logger"), ytdl_logger) + .wrap_exc(py)?; + } + + let inner = { + let p_params = opts.into_bound_py_any(py).wrap_exc(py)?; + let p_auto_init = true.into_bound_py_any(py).wrap_exc(py)?; + + py.import(intern!(py, "yt_dlp.YoutubeDL")) + .wrap_exc(py)? + .getattr(intern!(py, "YoutubeDL")) + .wrap_exc(py)? + .call1( + PyTuple::new( + py, + [ + p_params.into_bound_py_any(py).wrap_exc(py)?, + p_auto_init.into_bound_py_any(py).wrap_exc(py)?, + ], + ) + .wrap_exc(py)?, + ) + .wrap_exc(py)? + }; + + { + // Setup the post processors + let add_post_processor_fun = inner + .getattr(intern!(py, "add_post_processor")) + .wrap_exc(py)?; + + for pp in options.post_processors { + add_post_processor_fun + .call( + (pp(py).wrap_exc(py)?.into_bound_py_any(py).wrap_exc(py)?,), + // "when" can take any value in yt_dlp.utils.POSTPROCESS_WHEN + py_kw_args!(py => when = "pre_process"), + ) + .wrap_exc(py)?; + } + } + + Ok::<_, PythonError>(inner.unbind()) + })?; + + Ok(Self { + inner: yt_dlp_module, + options: output_options, + }) + } +} + +#[allow(missing_docs)] +pub mod build { + use crate::python_error::PythonError; + + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + } +} diff --git a/crates/yt_dlp/src/post_processors/dearrow.rs b/crates/yt_dlp/src/post_processors/dearrow.rs new file mode 100644 index 0000000..7787d68 --- /dev/null +++ b/crates/yt_dlp/src/post_processors/dearrow.rs @@ -0,0 +1,247 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use curl::easy::Easy; +use log::{error, info, trace, warn}; +use pyo3::{ + Bound, PyAny, PyErr, PyResult, Python, exceptions, intern, pyfunction, + types::{PyAnyMethods, PyDict, PyModule}, + wrap_pyfunction, +}; +use serde::{Deserialize, Serialize}; + +use crate::{ + pydict_cast, pydict_get, + python_error::{IntoPythonError, PythonError}, +}; + +/// # Errors +/// - If the underlying function returns an error. +/// - If python operations fail. +pub fn process(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> { + #[pyfunction] + fn actual_processor(info_json: Bound<'_, PyDict>) -> PyResult<Bound<'_, PyDict>> { + let output = match unwrapped_process(info_json) { + Ok(ok) => ok, + Err(err) => { + return Err(PyErr::new::<exceptions::PyRuntimeError, _>(err.to_string())); + } + }; + Ok(output) + } + + let module = PyModule::new(py, "rust_post_processors")?; + let scope = PyDict::new(py); + scope.set_item( + intern!(py, "actual_processor"), + wrap_pyfunction!(actual_processor, module)?, + )?; + py.run( + c" +import yt_dlp + +class DeArrow(yt_dlp.postprocessor.PostProcessor): + def run(self, info): + info = actual_processor(info) + return [], info + +inst = DeArrow() +", + Some(&scope), + None, + )?; + + Ok(scope.get_item(intern!(py, "inst"))?.cast_into()?) +} + +/// # Errors +/// If the API access fails. +pub fn unwrapped_process(info: Bound<'_, PyDict>) -> Result<Bound<'_, PyDict>, Error> { + if pydict_get!(info, "extractor_key", String).as_str() != "Youtube" { + return Ok(info); + } + + let mut retry_num = 3; + let mut output: DeArrowApi = { + loop { + let output_bytes = { + let mut dst = Vec::new(); + + let mut easy = Easy::new(); + easy.url( + format!( + "https://sponsor.ajay.app/api/branding?videoID={}", + pydict_get!(info, "id", String) + ) + .as_str(), + )?; + + let mut transfer = easy.transfer(); + transfer.write_function(|data| { + dst.extend_from_slice(data); + Ok(data.len()) + })?; + transfer.perform()?; + drop(transfer); + + dst + }; + + match serde_json::from_slice(&output_bytes) { + Ok(ok) => break ok, + Err(err) => { + if retry_num > 0 { + trace!( + "DeArrow: Api access failed, trying again ({retry_num} retries left)" + ); + retry_num -= 1; + } else { + let err: serde_json::Error = err; + return Err(err.into()); + } + } + } + } + }; + + // We pop the titles, so we need this vector reversed. + output.titles.reverse(); + + let title_len = output.titles.len(); + let mut iterator = output.titles.clone(); + let selected = loop { + let Some(title) = iterator.pop() else { + break false; + }; + + if (title.locked || title.votes < 1) && title_len > 1 { + info!( + "DeArrow: Skipping title {:#?}, as it is not good enough", + title.value + ); + // Skip titles that are not “good” enough. + continue; + } + + update_title(&info, &title.value).wrap_exc(info.py())?; + + break true; + }; + + if !selected && title_len != 0 { + // No title was selected, even though we had some titles. + // Just pick the first one in this case. + update_title(&info, &output.titles[0].value).wrap_exc(info.py())?; + } + + Ok(info) +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + + #[error("Failed to access the DeArrow api: {0}")] + Get(#[from] curl::Error), + + #[error("Failed to deserialize a api json return object: {0}")] + Deserialize(#[from] serde_json::Error), +} + +fn update_title(info: &Bound<'_, PyDict>, new_title: &str) -> PyResult<()> { + let py = info.py(); + + assert!(!info.contains(intern!(py, "original_title"))?); + + if let Ok(old_title) = info.get_item(intern!(py, "title")) { + warn!( + "DeArrow: Updating title from {:#?} to {:#?}", + pydict_cast!(old_title, &str), + new_title + ); + + info.set_item(intern!(py, "original_title"), old_title) + .expect("We checked, it is a new key"); + } else { + warn!("DeArrow: Setting title to {new_title:#?}"); + } + + let cleaned_title = { + // NOTE(@bpeetz): DeArrow uses `>` as a “Don't format the next word” mark. + // They should be removed, if one does not use a auto-formatter. <2025-06-16> + new_title.replace('>', "") + }; + + info.set_item(intern!(py, "title"), cleaned_title) + .expect("This should work?"); + + Ok(()) +} + +#[derive(Serialize, Deserialize)] +/// See: <https://wiki.sponsor.ajay.app/w/API_Docs/DeArrow> +struct DeArrowApi { + titles: Vec<Title>, + thumbnails: Vec<Thumbnail>, + + #[serde(alias = "randomTime")] + random_time: Option<f64>, + + #[serde(alias = "videoDuration")] + video_duration: Option<f64>, + + #[serde(alias = "casualVotes")] + casual_votes: Vec<CasualVote>, +} + +#[derive(Serialize, Deserialize)] +struct CasualVote { + id: String, + count: u32, + title: String, +} + +#[derive(Serialize, Deserialize, Clone)] +struct Title { + /// Note: Titles will sometimes contain > before a word. + /// This tells the auto-formatter to not format a word. + /// If you have no auto-formatter, you can ignore this and replace it with an empty string + #[serde(alias = "title")] + value: String, + + original: bool, + votes: u64, + locked: bool, + + #[serde(alias = "UUID")] + uuid: String, + + /// only present if requested + #[serde(alias = "userID")] + user_id: Option<String>, +} + +#[derive(Serialize, Deserialize)] +struct Thumbnail { + // null if original is true + timestamp: Option<f64>, + + original: bool, + votes: u64, + locked: bool, + + #[serde(alias = "UUID")] + uuid: String, + + /// only present if requested + #[serde(alias = "userID")] + user_id: Option<String>, +} diff --git a/crates/yt_dlp/src/post_processors/mod.rs b/crates/yt_dlp/src/post_processors/mod.rs new file mode 100644 index 0000000..d9be3f5 --- /dev/null +++ b/crates/yt_dlp/src/post_processors/mod.rs @@ -0,0 +1,48 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub mod dearrow; + +#[macro_export] +macro_rules! pydict_get { + ($value:expr, $name:literal, $into:ty) => {{ + let item = $value.get_item(pyo3::intern!($value.py(), $name)); + match &item { + Ok(val) => $crate::pydict_cast!(val, $into), + Err(_) => panic!( + concat!( + "Expected '", + $name, + "' to be a key for the'", + stringify!($value), + "' py dictionary: {:#?}" + ), + $value + ), + } + }}; +} + +#[macro_export] +macro_rules! pydict_cast { + ($value:expr, $into:ty) => {{ + match $value.extract::<$into>() { + Ok(result) => result, + Err(val) => panic!( + concat!( + "Expected to be able to extract ", + stringify!($into), + " from value ({:#?})." + ), + val + ), + } + }}; +} diff --git a/crates/yt_dlp/src/progress_hook.rs b/crates/yt_dlp/src/progress_hook.rs index 7a7628a..7e5f8a5 100644 --- a/crates/yt_dlp/src/progress_hook.rs +++ b/crates/yt_dlp/src/progress_hook.rs @@ -1,41 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + #[macro_export] -macro_rules! mk_python_function { +macro_rules! wrap_progress_hook { ($name:ident, $new_name:ident) => { - pub fn $new_name( - mut args: $crate::progress_hook::rustpython::vm::function::FuncArgs, - vm: &$crate::progress_hook::rustpython::vm::VirtualMachine, - ) { - use $crate::progress_hook::rustpython; - - let input = { - let dict: rustpython::vm::PyRef<rustpython::vm::builtins::PyDict> = args - .args - .remove(0) - .downcast() - .expect("The progress hook is always called with these args"); - let new_dict = rustpython::vm::builtins::PyDict::new_ref(&vm.ctx); - dict.into_iter() - .filter_map(|(name, value)| { - let real_name: rustpython::vm::PyRefExact<rustpython::vm::builtins::PyStr> = - name.downcast_exact(vm).expect("Is a string"); - let name_str = real_name.to_str().expect("Is a string"); - if name_str.starts_with('_') { - None - } else { - Some((name_str.to_owned(), value)) - } - }) - .for_each(|(key, value)| { - new_dict - .set_item(&key, value, vm) - .expect("This is a transpositions, should always be valid"); - }); - - $crate::json_dumps(new_dict, vm) - }; - $name(input).expect("Shall not fail!"); + pub(crate) fn $new_name( + py: yt_dlp::progress_hook::__priv::pyo3::Python<'_>, + ) -> yt_dlp::progress_hook::__priv::pyo3::PyResult< + yt_dlp::progress_hook::__priv::pyo3::Bound< + '_, + yt_dlp::progress_hook::__priv::pyo3::types::PyCFunction, + >, + > { + #[yt_dlp::progress_hook::__priv::pyo3::pyfunction] + #[pyo3(crate = "yt_dlp::progress_hook::__priv::pyo3")] + fn inner( + input: yt_dlp::progress_hook::__priv::pyo3::Bound< + '_, + yt_dlp::progress_hook::__priv::pyo3::types::PyDict, + >, + ) -> yt_dlp::progress_hook::__priv::pyo3::PyResult<()> { + let processed_input = { + let new_dict = yt_dlp::progress_hook::__priv::pyo3::types::PyDict::new(input.py()); + + input + .into_iter() + .filter_map(|(name, value)| { + let real_name = yt_dlp::progress_hook::__priv::pyo3::types::PyAnyMethods::extract::<String>(&name).expect("Should always be a string"); + + if real_name.starts_with('_') { + None + } else { + Some((real_name, value)) + } + }) + .for_each(|(key, value)| { + yt_dlp::progress_hook::__priv::pyo3::types::PyDictMethods::set_item(&new_dict, &key, value) + .expect("This is a transpositions, should always be valid"); + }); + yt_dlp::progress_hook::__priv::json_dumps(&new_dict) + }; + + $name(processed_input)?; + + Ok(()) + } + + let module = yt_dlp::progress_hook::__priv::pyo3::types::PyModule::new(py, "progress_hook")?; + let fun = yt_dlp::progress_hook::__priv::pyo3::wrap_pyfunction!(inner, module)?; + + Ok(fun) } }; } -pub use rustpython; +pub mod __priv { + pub use crate::info_json::{json_dumps, json_loads}; + pub use pyo3; +} diff --git a/crates/yt_dlp/src/python_error.rs b/crates/yt_dlp/src/python_error.rs new file mode 100644 index 0000000..0c442b3 --- /dev/null +++ b/crates/yt_dlp/src/python_error.rs @@ -0,0 +1,55 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{self, Display}; + +use log::{Level, debug, log_enabled}; +use pyo3::{PyErr, Python, types::PyTracebackMethods}; + +#[derive(thiserror::Error, Debug)] +pub struct PythonError(pub String); + +pub(crate) trait IntoPythonError<T>: Sized { + fn wrap_exc(self, py: Python<'_>) -> Result<T, PythonError>; +} + +impl<T> IntoPythonError<T> for Result<T, PyErr> { + fn wrap_exc(self, py: Python<'_>) -> Result<T, PythonError> { + self.map_err(|exc| PythonError::from_exception(py, &exc)) + } +} + +impl Display for PythonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Python threw an exception: {}", self.0) + } +} + +impl PythonError { + pub(super) fn from_exception(py: Python<'_>, exc: &PyErr) -> Self { + let buffer = process_exception(py, exc); + Self(buffer) + } +} + +pub(super) fn process_exception(py: Python<'_>, err: &PyErr) -> String { + if log_enabled!(Level::Debug) { + let mut output = err.to_string(); + + if let Some(tb) = err.traceback(py) { + output.push('\n'); + output.push_str(&tb.format().unwrap()); + } + + debug!("Python threw an exception: {output}"); + } + + err.to_string() +} diff --git a/crates/yt_dlp/update.sh b/crates/yt_dlp/update.sh index c1a0215..52c96b5 100755 --- a/crates/yt_dlp/update.sh +++ b/crates/yt_dlp/update.sh @@ -10,6 +10,4 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -cd "$(dirname "$0")" || exit 1 -[ "$1" = "upgrade" ] && cargo upgrade --incompatible -cargo update +"$(dirname "$0")/crates/pyo3-pylogger/update.sh" "$@" |