diff options
Diffstat (limited to 'crates')
167 files changed, 14671 insertions, 1643 deletions
diff --git a/crates/bytes/Cargo.lock b/crates/bytes/Cargo.lock deleted file mode 100644 index b30ba3d..0000000 --- a/crates/bytes/Cargo.lock +++ /dev/null @@ -1,65 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "bytes" -version = "1.0.0" -dependencies = [ - "serde", -] - -[[package]] -name = "proc-macro2" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "serde" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "syn" -version = "2.0.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" diff --git a/crates/bytes/src/serde.rs b/crates/bytes/src/serde.rs deleted file mode 100644 index 4341e32..0000000 --- a/crates/bytes/src/serde.rs +++ /dev/null @@ -1,19 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use serde::{Serialize, Serializer}; - -use crate::Bytes; - -impl Serialize for Bytes { - fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { - serializer.serialize_str(self.to_string().as_str()) - } -} diff --git a/crates/bytes/Cargo.toml b/crates/colors/Cargo.toml index 4439aa8..4edefcf 100644 --- a/crates/bytes/Cargo.toml +++ b/crates/colors/Cargo.toml @@ -1,7 +1,8 @@ # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 uutils developers +# SPDX-License-Identifier: MIT # # This file is part of Yt. # @@ -9,25 +10,17 @@ # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. [package] -name = "bytes" -description = "Simple byte formatting utilities" -keywords = [] -categories = [] -version.workspace = true -edition.workspace = true +name = "colors" authors.workspace = true license.workspace = true +description = "A owo-colors inspired color crate." +version.workspace = true +edition.workspace = true repository.workspace = true rust-version.workspace = true publish = false [dependencies] -serde.workspace = true - -[dev-dependencies] [lints] workspace = true - -[package.metadata.docs.rs] -all-features = true diff --git a/crates/colors/src/custom.rs b/crates/colors/src/custom.rs new file mode 100644 index 0000000..fd6b7b3 --- /dev/null +++ b/crates/colors/src/custom.rs @@ -0,0 +1,75 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// Taken from <https://github.com/owo-colors/owo-colors/blob/61f8bba2f5f80e9f4fa600fbfdf2c21656f1d523/src/colors/custom.rs> +// at 2025-07-16T18:05:55 CEST. + +const U8_TO_STR: [[u8; 3]; 256] = generate_lookup(); + +const fn generate_lookup() -> [[u8; 3]; 256] { + let mut table = [[0, 0, 0]; 256]; + + let mut i = 0; + while i < 256 { + table[i] = [ + b'0' + (i / 100) as u8, + b'0' + (i / 10 % 10) as u8, + b'0' + (i % 10) as u8, + ]; + i += 1; + } + + table +} + +#[derive(Clone, Copy)] +pub(crate) enum Plane { + Fg, + Bg, +} + +pub(crate) const fn rgb_to_ansi(r: u8, g: u8, b: u8, plane: Plane) -> [u8; 18] { + let mut buf = *b"\x1b[p8;2;rrr;ggg;bbb"; + + let r = U8_TO_STR[r as usize]; + let g = U8_TO_STR[g as usize]; + let b = U8_TO_STR[b as usize]; + + // p 2 + buf[2] = match plane { + Plane::Fg => b'3', + Plane::Bg => b'4', + }; + + // r 7 + buf[7] = r[0]; + buf[8] = r[1]; + buf[9] = r[2]; + + // g 11 + buf[11] = g[0]; + buf[12] = g[1]; + buf[13] = g[2]; + + // b 15 + buf[15] = b[0]; + buf[16] = b[1]; + buf[17] = b[2]; + + buf +} + +/// This exists since [`unwrap()`] isn't const-safe (it invokes formatting infrastructure) +pub(crate) const fn bytes_to_str(bytes: &'static [u8]) -> &'static str { + match core::str::from_utf8(bytes) { + Ok(o) => o, + Err(_e) => panic!("Const parsing &[u8] to a string failed!"), + } +} diff --git a/crates/colors/src/lib.rs b/crates/colors/src/lib.rs new file mode 100644 index 0000000..663e19a --- /dev/null +++ b/crates/colors/src/lib.rs @@ -0,0 +1,97 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{Display, Write}; + +use crate::{ + list::{elements, methods}, + support::{CSE, CSI, elements_inner}, +}; + +pub(crate) mod custom; +mod list; +mod support; + +#[derive(Debug)] +pub struct Canvas<I: Display>(I); + +impl<I: Display> Colorize for Canvas<I> { + fn render_into(self, base: &mut String, use_colors: bool) { + write!(base, "{}", self.0).expect("Is written into a string"); + + if use_colors { + // Reset the color and style, if we used colours. + base.write_str(CSI).expect("In-memory write"); + base.write_str("0").expect("In-memory write"); + base.write_str(CSE).expect("In-memory write"); + } + } +} + +pub trait IntoCanvas: Display + Sized { + fn into_canvas(self) -> Canvas<Self> { + Canvas(self) + } + + methods! { IntoCanvas } +} + +impl<I: Display> IntoCanvas for I {} + +pub trait Colorize: Sized { + /// Turn this colorized struct into a string, by writing into the base. + fn render_into(self, base: &mut String, use_colors: bool); + + /// Turn this colorized struct into a string for consumption. + fn render(self, use_colors: bool) -> String { + let mut base = String::new(); + self.render_into(&mut base, use_colors); + base + } + + methods! { Colorize } +} + +elements! {} + +#[cfg(test)] +mod tests { + use crate::{Colorize, IntoCanvas}; + + #[test] + fn test_colorize_basic() { + let base = "Base".green().render(true); + #[rustfmt::skip] + let expected = concat!( + "\x1b[32m", + "Base", + "\x1b[0m", + ); + + assert_eq!(base.as_str(), expected); + } + + #[test] + fn test_colorize_combo() { + let base = "Base".green().on_red().bold().strike_through().render(true); + + #[rustfmt::skip] + let expected = concat!( + "\x1b[9m", // strike_through + "\x1b[1m", // bold + "\x1b[41m", // on_red + "\x1b[32m", // green + "Base", + "\x1b[0m", + ); + + assert_eq!(base.as_str(), expected); + } +} diff --git a/crates/colors/src/list.rs b/crates/colors/src/list.rs new file mode 100644 index 0000000..35fcb83 --- /dev/null +++ b/crates/colors/src/list.rs @@ -0,0 +1,233 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::support::prepend_input; + +prepend_input! { + crate::support::methods_inner as methods (($tt:tt) -> {$tt}), + crate::support::elements_inner as elements, + + <shared_input> + { + // Colors + Black black 30, + OnBlack on_black 40, + + Red red 31, + OnRed on_red 41, + + Green green 32, + OnGreen on_green 42, + + Yellow yellow 33, + OnYellow on_yellow 43, + + Blue blue 34, + OnBlue on_blue 44, + + Magenta magenta 35, + OnMagenta on_magenta 45, + + Cyan cyan 36, + OnCyan on_cyan 46, + + White white 37, + OnWhite on_white 47, + + Default default 39, + OnDefault on_default 49, + + // Bright bright colors + BrightBlack bright_black 90, + OnBrightBlack on_bright_black 100, + + BrightRed bright_red 91, + OnBrightRed on_bright_red 101, + + BrightGreen bright_green 92, + OnBrightGreen on_bright_green 102, + + BrightYellow bright_yellow 93, + OnBrightYellow on_bright_yellow 103, + + BrightBlue bright_blue 94, + OnBrightBlue on_bright_blue 104, + + BrightMagenta bright_magenta 95, + OnBrightMagenta on_bright_magenta 105, + + BrightCyan bright_cyan 96, + OnBrightCyan on_bright_cyan 106, + + BrightWhite bright_white 97, + OnBrightWhite on_bright_white 107, + + // CSS colors + // TODO(@bpeetz): Also support background colors with these values. <2025-07-16> + AliceBlue alice_blue (240, 248, 255), + AntiqueWhite antique_white (250, 235, 215), + Aqua aqua (0, 255, 255), + Aquamarine aquamarine (127, 255, 212), + Azure azure (240, 255, 255), + Beige beige (245, 245, 220), + Bisque bisque (255, 228, 196), + // Black black (0, 0, 0), + BlanchedAlmond blanched_almond (255, 235, 205), + // Blue blue (0, 0, 255), + BlueViolet blue_violet (138, 43, 226), + Brown brown (165, 42, 42), + BurlyWood burly_wood (222, 184, 135), + CadetBlue cadet_blue (95, 158, 160), + Chartreuse chartreuse (127, 255, 0), + Chocolate chocolate (210, 105, 30), + Coral coral (255, 127, 80), + CornflowerBlue cornflower_blue (100, 149, 237), + Cornsilk cornsilk (255, 248, 220), + Crimson crimson (220, 20, 60), + DarkBlue dark_blue (0, 0, 139), + DarkCyan dark_cyan (0, 139, 139), + DarkGoldenRod dark_golden_rod (184, 134, 11), + DarkGray dark_gray (169, 169, 169), + DarkGrey dark_grey (169, 169, 169), + DarkGreen dark_green (0, 100, 0), + DarkKhaki dark_khaki (189, 183, 107), + DarkMagenta dark_magenta (139, 0, 139), + DarkOliveGreen dark_olive_green (85, 107, 47), + DarkOrange dark_orange (255, 140, 0), + DarkOrchid dark_orchid (153, 50, 204), + DarkRed dark_red (139, 0, 0), + DarkSalmon dark_salmon (233, 150, 122), + DarkSeaGreen dark_sea_green (143, 188, 143), + DarkSlateBlue dark_slate_blue (72, 61, 139), + DarkSlateGray dark_slate_gray (47, 79, 79), + DarkSlateGrey dark_slate_grey (47, 79, 79), + DarkTurquoise dark_turquoise (0, 206, 209), + DarkViolet dark_violet (148, 0, 211), + DeepPink deep_pink (255, 20, 147), + DeepSkyBlue deep_sky_blue (0, 191, 255), + DimGray dim_gray (105, 105, 105), + DimGrey dim_grey (105, 105, 105), + DodgerBlue dodger_blue (30, 144, 255), + FireBrick fire_brick (178, 34, 34), + FloralWhite floral_white (255, 250, 240), + ForestGreen forest_green (34, 139, 34), + Fuchsia fuchsia (255, 0, 255), + Gainsboro gainsboro (220, 220, 220), + GhostWhite ghost_white (248, 248, 255), + Gold gold (255, 215, 0), + GoldenRod golden_rod (218, 165, 32), + Gray gray (128, 128, 128), + Grey grey (128, 128, 128), + // Green green (0, 128, 0), + GreenYellow green_yellow (173, 255, 47), + HoneyDew honey_dew (240, 255, 240), + HotPink hot_pink (255, 105, 180), + IndianRed indian_red (205, 92, 92), + Indigo indigo (75, 0, 130), + Ivory ivory (255, 255, 240), + Khaki khaki (240, 230, 140), + Lavender lavender (230, 230, 250), + LavenderBlush lavender_blush (255, 240, 245), + LawnGreen lawn_green (124, 252, 0), + LemonChiffon lemon_chiffon (255, 250, 205), + LightBlue light_blue (173, 216, 230), + LightCoral light_coral (240, 128, 128), + LightCyan light_cyan (224, 255, 255), + LightGoldenRodYellow light_golden_rod_yellow (250, 250, 210), + LightGray light_gray (211, 211, 211), + LightGrey light_grey (211, 211, 211), + LightGreen light_green (144, 238, 144), + LightPink light_pink (255, 182, 193), + LightSalmon light_salmon (255, 160, 122), + LightSeaGreen light_sea_green (32, 178, 170), + LightSkyBlue light_sky_blue (135, 206, 250), + LightSlateGray light_slate_gray (119, 136, 153), + LightSlateGrey light_slate_grey (119, 136, 153), + LightSteelBlue light_steel_blue (176, 196, 222), + LightYellow light_yellow (255, 255, 224), + Lime lime (0, 255, 0), + LimeGreen lime_green (50, 205, 50), + Linen linen (250, 240, 230), + // Magenta magenta (255, 0, 255), + Maroon maroon (128, 0, 0), + MediumAquaMarine medium_aqua_marine (102, 205, 170), + MediumBlue medium_blue (0, 0, 205), + MediumOrchid medium_orchid (186, 85, 211), + MediumPurple medium_purple (147, 112, 219), + MediumSeaGreen medium_sea_green (60, 179, 113), + MediumSlateBlue medium_slate_blue (123, 104, 238), + MediumSpringGreen medium_spring_green (0, 250, 154), + MediumTurquoise medium_turquoise (72, 209, 204), + MediumVioletRed medium_violet_red (199, 21, 133), + MidnightBlue midnight_blue (25, 25, 112), + MintCream mint_cream (245, 255, 250), + MistyRose misty_rose (255, 228, 225), + Moccasin moccasin (255, 228, 181), + NavajoWhite navajo_white (255, 222, 173), + Navy navy (0, 0, 128), + OldLace old_lace (253, 245, 230), + Olive olive (128, 128, 0), + OliveDrab olive_drab (107, 142, 35), + Orange orange (255, 165, 0), + OrangeRed orange_red (255, 69, 0), + Orchid orchid (218, 112, 214), + PaleGoldenRod pale_golden_rod (238, 232, 170), + PaleGreen pale_green (152, 251, 152), + PaleTurquoise pale_turquoise (175, 238, 238), + PaleVioletRed pale_violet_red (219, 112, 147), + PapayaWhip papaya_whip (255, 239, 213), + PeachPuff peach_puff (255, 218, 185), + Peru peru (205, 133, 63), + Pink pink (255, 192, 203), + Plum plum (221, 160, 221), + PowderBlue powder_blue (176, 224, 230), + Purple purple (128, 0, 128), + RebeccaPurple rebecca_purple (102, 51, 153), + // Red red (255, 0, 0), + RosyBrown rosy_brown (188, 143, 143), + RoyalBlue royal_blue (65, 105, 225), + SaddleBrown saddle_brown (139, 69, 19), + Salmon salmon (250, 128, 114), + SandyBrown sandy_brown (244, 164, 96), + SeaGreen sea_green (46, 139, 87), + SeaShell sea_shell (255, 245, 238), + Sienna sienna (160, 82, 45), + Silver silver (192, 192, 192), + SkyBlue sky_blue (135, 206, 235), + SlateBlue slate_blue (106, 90, 205), + SlateGray slate_gray (112, 128, 144), + SlateGrey slate_grey (112, 128, 144), + Snow snow (255, 250, 250), + SpringGreen spring_green (0, 255, 127), + SteelBlue steel_blue (70, 130, 180), + Tan tan (210, 180, 140), + Teal teal (0, 128, 128), + Thistle thistle (216, 191, 216), + Tomato tomato (255, 99, 71), + Turquoise turquoise (64, 224, 208), + Violet violet (238, 130, 238), + Wheat wheat (245, 222, 179), + // White white (255, 255, 255), + WhiteSmoke white_smoke (245, 245, 245), + // Yellow yellow (255, 255, 0), + YellowGreen yellow_green (154, 205, 50), + + // Styles + Bold bold 1, + Dim dim 2, + Italic italic 3, + Underline underline 4, + Blink blink 5, + BlinkFast blink_fast 6, + Reversed reversed 7, + Hidden hidden 8, + StrikeThrough strike_through 9, + } +} diff --git a/crates/colors/src/support.rs b/crates/colors/src/support.rs new file mode 100644 index 0000000..3c3f87d --- /dev/null +++ b/crates/colors/src/support.rs @@ -0,0 +1,126 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(super) const CSI: &str = "\x1b["; +pub(super) const CSE: &str = "m"; + +macro_rules! elements_inner { + ( + $( + $name:ident $_:ident $number:tt + ),* + $(,)? + ) => { + $( + #[derive(Debug)] + pub struct $name<I: Colorize>(I); + + impl<I: Colorize> Colorize for $name<I> { + fn render_into(self, base: &mut String, use_colors: bool) { + elements_inner! {@parse_number $number} + + if use_colors { + base.write_str(CSI).expect("In-memory write"); + base.write_str(NUMBERS).expect("In-memory write"); + base.write_str(CSE).expect("In-memory write"); + } + self.0.render_into(base, use_colors); + // The canvas is resetting the colours again. + } + } + )* + }; + + (@parse_number $single:literal) => { + const NUMBERS: &str = stringify!($single); + }; + (@parse_number ($red:literal, $green:literal, $blue:literal)) => { + const NUMBERS_U8: [u8; 18] = $crate::custom::rgb_to_ansi($red, $green, $blue, $crate::custom::Plane::Fg); + + const NUMBERS: &str = $crate::custom::bytes_to_str(&NUMBERS_U8); + } +} +pub(super) use elements_inner; + +macro_rules! methods_inner { + ( + Colorize + + $( + $struct_name:ident $fn_name:ident $_:tt + ),* + $(,)? + ) => { + $( + fn $fn_name(self) -> $struct_name<Self> { + $struct_name(self) + } + )* + }; + ( + IntoCanvas + + $( + $struct_name:ident $fn_name:ident $_:tt + ),* + $(,)? + ) => { + $( + fn $fn_name(self) -> $struct_name<Canvas<Self>> { + $struct_name(Canvas(self)) + } + )* + }; +} +pub(super) use methods_inner; + +macro_rules! prepend_input { + ( + $( + $existing_macro_name:path as $new_macro_name:ident $(($macro_rule:tt -> $macro_apply:tt))? + ),* + $(,)? + + <shared_input> + $shared_input:tt + ) => { + $( + prepend_input! { + @generate_macro + $existing_macro_name as $new_macro_name $(($macro_rule -> $macro_apply))? + <shared_input> + $shared_input + } + )* + }; + + ( + @generate_macro + $existing_macro_name:path as $new_macro_name:ident $((($($macro_rule:tt)*) -> {$($macro_apply:tt)*}))? + + <shared_input> + { + $( + $shared_input:tt + )* + } + ) => { + macro_rules! $new_macro_name { + ($($($macro_rule)*)?) => { + $existing_macro_name! { + $($($macro_apply)*)? + $($shared_input)* + } + } + } + pub(super) use $new_macro_name; + } +} +pub(crate) use prepend_input; diff --git a/crates/fmt/Cargo.toml b/crates/fmt/Cargo.toml new file mode 100644 index 0000000..f3cf4ad --- /dev/null +++ b/crates/fmt/Cargo.toml @@ -0,0 +1,30 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 uutils developers +# SPDX-License-Identifier: MIT +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +[package] +name = "uu_fmt" +authors = ["uutils developers", "Benedikt Peetz <benedikt.peetz@b-peetz.de>"] +license = "MIT" +description = "A fork of the uutils fmt tool. This fork is a library instead of a binary." +version.workspace = true +edition.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +path = "src/fmt.rs" + +[dependencies] +unicode-width = "0.2.1" + +[lints] +workspace = true diff --git a/crates/fmt/LICENSE b/crates/fmt/LICENSE new file mode 100644 index 0000000..21bd444 --- /dev/null +++ b/crates/fmt/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) uutils developers + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/fmt/LICENSE.license b/crates/fmt/LICENSE.license new file mode 100644 index 0000000..6cee99d --- /dev/null +++ b/crates/fmt/LICENSE.license @@ -0,0 +1,10 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +Copyright (C) 2025 uutils developers +SPDX-License-Identifier: MIT + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/crates/fmt/src/fmt.rs b/crates/fmt/src/fmt.rs new file mode 100644 index 0000000..3067bea --- /dev/null +++ b/crates/fmt/src/fmt.rs @@ -0,0 +1,137 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 uutils developers +// SPDX-License-Identifier: MIT +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::fmt::Write; + +use linebreak::break_lines; +use parasplit::ParagraphStream; + +mod linebreak; +mod parasplit; + +#[derive(Debug)] +#[allow(clippy::struct_excessive_bools)] +pub struct FmtOptions { + /// First and second line of paragraph + /// may have different indentations, in which + /// case the first line's indentation is preserved, + /// and each subsequent line's indentation matches the second line. + pub crown_margin: bool, + + /// Like the [`crown_margin`], except that the first and second line of a paragraph *must* + /// have different indentation or they are treated as separate paragraphs. + pub tagged_paragraph: bool, + + /// Attempt to detect and preserve mail headers in the input. + /// Be careful when combining this with [`prefix`]. + pub mail: bool, + + /// Split lines only, do not reflow. + pub split_only: bool, + + /// Insert exactly one space between words, and two between sentences. + /// Sentence breaks in the input are detected as [?!.] followed by two spaces or a newline; + /// other punctuation is not interpreted as a sentence break. + pub uniform: bool, + + /// Reformat only lines beginning with PREFIX, reattaching PREFIX to reformatted lines. + /// Unless [`exact_prefix`] is specified, leading whitespace will be ignored when matching PREFIX. + pub prefix: Option<String>, + + /// Do not reformat lines beginning with ``ANTI_PREFIX``. + /// Unless [`exact_anti_prefix`] is specified, leading whitespace will be ignored when matching ``ANTI_PREFIX``. + pub anti_prefix: Option<String>, + + /// [`prefix`] must match at the beginning of the line with no preceding whitespace. + pub exact_prefix: bool, + + /// [`anti_prefix`] must match at the beginning of the line with no preceding whitespace. + pub exact_anti_prefix: bool, + + /// Fill output lines up to a maximum of WIDTH columns, default 75. + pub width: usize, + + /// Goal width, default of 93% of WIDTH. + /// Must be less than or equal to WIDTH. + pub goal: usize, + + /// Break lines more quickly at the expense of a potentially more ragged appearance. + pub quick: bool, + + /// Treat tabs as TABWIDTH spaces for determining line length, default 8. + /// Note that this is used only for calculating line lengths; tabs are preserved in the output. + pub tabwidth: usize, +} + +impl FmtOptions { + #[must_use] + #[allow(clippy::cast_sign_loss)] + #[allow(clippy::cast_possible_truncation)] + #[allow(clippy::cast_precision_loss)] + pub fn new(width: Option<usize>, goal: Option<usize>, tabwidth: Option<usize>) -> Self { + // by default, goal is 93% of width + const DEFAULT_GOAL_TO_WIDTH_RATIO: f64 = 0.93; + const DEFAULT_WIDTH: usize = 75; + + FmtOptions { + crown_margin: false, + tagged_paragraph: false, + mail: false, + split_only: false, + uniform: false, + prefix: None, + anti_prefix: None, + exact_prefix: false, + exact_anti_prefix: false, + width: width.unwrap_or(DEFAULT_WIDTH), + goal: goal.unwrap_or( + ((width.unwrap_or(DEFAULT_WIDTH) as f64) * DEFAULT_GOAL_TO_WIDTH_RATIO).floor() + as usize, + ), + quick: false, + tabwidth: tabwidth.unwrap_or(8), + } + } +} + +/// Process text and format it according to the provided options. +/// +/// # Arguments +/// +/// * `text` - The text to process. +/// * `fmt_opts` - A reference to a [`FmtOptions`] structure containing the formatting options. +/// +/// # Returns +/// +/// The formatted [`String`]. +#[must_use] +pub fn process_text(text: &str, fmt_opts: &FmtOptions) -> String { + let mut output = String::new(); + + let p_stream = ParagraphStream::new(fmt_opts, text); + for para_result in p_stream { + match para_result { + Err(s) => { + output.push_str(&s); + output.push('\n'); + } + Ok(para) => write!(output, "{}", break_lines(¶, fmt_opts)) + .expect("This is in-memory. It should not fail"), + } + } + + output +} diff --git a/crates/fmt/src/linebreak.rs b/crates/fmt/src/linebreak.rs new file mode 100644 index 0000000..b1dc6fa --- /dev/null +++ b/crates/fmt/src/linebreak.rs @@ -0,0 +1,520 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 uutils developers +// SPDX-License-Identifier: MIT +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::fmt::Write; +use std::{cmp, mem}; + +use crate::FmtOptions; +use crate::parasplit::{ParaWords, Paragraph, WordInfo}; + +struct BreakArgs<'a> { + opts: &'a FmtOptions, + init_len: usize, + indent_str: &'a str, + indent_len: usize, + uniform: bool, + output: String, +} + +impl BreakArgs<'_> { + fn compute_width(&self, winfo: &WordInfo<'_>, position_n: usize, fresh: bool) -> usize { + if fresh { + 0 + } else { + let post = winfo.after_tab; + match winfo.before_tab { + None => post, + Some(pre) => { + post + ((pre + position_n) / self.opts.tabwidth + 1) * self.opts.tabwidth + - position_n + } + } + } + } +} + +pub(super) fn break_lines(para: &Paragraph, opts: &FmtOptions) -> String { + let mut output = String::new(); + + // indent + let p_indent = ¶.indent_str; + let p_indent_len = para.indent_len; + + // words + let p_words = ParaWords::new(opts, para); + let mut p_words_words = p_words.words(); + + // the first word will *always* appear on the first line + // make sure of this here + let Some(winfo) = p_words_words.next() else { + return "\n".to_owned(); + }; + + // print the init, if it exists, and get its length + let p_init_len = winfo.word_nchars + + if opts.crown_margin || opts.tagged_paragraph { + // handle "init" portion + output.push_str(¶.init_str); + para.init_len + } else if !para.mail_header { + // for non-(crown, tagged) that's the same as a normal indent + output.push_str(p_indent); + p_indent_len + } else { + // except that mail headers get no indent at all + 0 + }; + + // write first word after writing init + write!(output, "{}", winfo.word).expect("Works"); + + // does this paragraph require uniform spacing? + let uniform = para.mail_header || opts.uniform; + + let mut break_args = BreakArgs { + opts, + init_len: p_init_len, + indent_str: p_indent, + indent_len: p_indent_len, + uniform, + output, + }; + + if opts.quick || para.mail_header { + break_simple(p_words_words, &mut break_args); + } else { + break_knuth_plass(p_words_words, &mut break_args); + }; + + break_args.output +} + +// break_simple implements a "greedy" breaking algorithm: print words until +// maxlength would be exceeded, then print a linebreak and indent and continue. +fn break_simple<'a, T: Iterator<Item = &'a WordInfo<'a>>>(iter: T, args: &mut BreakArgs<'a>) { + iter.fold((args.init_len, false), |(l, prev_punct), winfo| { + accum_words_simple(args, l, prev_punct, winfo) + }); + args.output.push('\n'); +} + +fn accum_words_simple<'a>( + args: &mut BreakArgs<'a>, + l: usize, + prev_punct: bool, + winfo: &'a WordInfo<'a>, +) -> (usize, bool) { + // compute the length of this word, considering how tabs will expand at this position on the line + let wlen = winfo.word_nchars + args.compute_width(winfo, l, false); + + let slen = compute_slen( + args.uniform, + winfo.new_line, + winfo.sentence_start, + prev_punct, + ); + + if l + wlen + slen > args.opts.width { + write_newline(args.indent_str, &mut args.output); + write_with_spaces(&winfo.word[winfo.word_start..], 0, &mut args.output); + (args.indent_len + winfo.word_nchars, winfo.ends_punct) + } else { + write_with_spaces(winfo.word, slen, &mut args.output); + (l + wlen + slen, winfo.ends_punct) + } +} + +// break_knuth_plass implements an "optimal" breaking algorithm in the style of +// Knuth, D.E., and Plass, M.F. "Breaking Paragraphs into Lines." in Software, +// Practice and Experience. Vol. 11, No. 11, November 1981. +// http://onlinelibrary.wiley.com/doi/10.1002/spe.4380111102/pdf +#[allow(trivial_casts)] +fn break_knuth_plass<'a, T: Clone + Iterator<Item = &'a WordInfo<'a>>>( + mut iter: T, + args: &mut BreakArgs<'a>, +) { + // run the algorithm to get the breakpoints + let breakpoints = find_kp_breakpoints(iter.clone(), args); + + // iterate through the breakpoints (note that breakpoints is in reverse break order, so we .rev() it + let result: (bool, bool) = breakpoints.iter().rev().fold( + (false, false), + |(mut prev_punct, mut fresh), &(next_break, break_before)| { + if fresh { + write_newline(args.indent_str, &mut args.output); + } + // at each breakpoint, keep emitting words until we find the word matching this breakpoint + for winfo in &mut iter { + let (slen, word) = slice_if_fresh( + fresh, + winfo.word, + winfo.word_start, + args.uniform, + winfo.new_line, + winfo.sentence_start, + prev_punct, + ); + fresh = false; + prev_punct = winfo.ends_punct; + + // We find identical breakpoints here by comparing addresses of the references. + // This is OK because the backing vector is not mutating once we are linebreaking. + let winfo_ptr = winfo as *const _; + let next_break_ptr = next_break as *const _; + if winfo_ptr == next_break_ptr { + // OK, we found the matching word + if break_before { + write_newline(args.indent_str, &mut args.output); + write_with_spaces(&winfo.word[winfo.word_start..], 0, &mut args.output); + } else { + // breaking after this word, so that means "fresh" is true for the next iteration + write_with_spaces(word, slen, &mut args.output); + fresh = true; + } + break; + } + write_with_spaces(word, slen, &mut args.output); + } + (prev_punct, fresh) + }, + ); + let (mut prev_punct, mut fresh) = result; + + // after the last linebreak, write out the rest of the final line. + for winfo in iter { + if fresh { + write_newline(args.indent_str, &mut args.output); + } + let (slen, word) = slice_if_fresh( + fresh, + winfo.word, + winfo.word_start, + args.uniform, + winfo.new_line, + winfo.sentence_start, + prev_punct, + ); + prev_punct = winfo.ends_punct; + fresh = false; + write_with_spaces(word, slen, &mut args.output); + } + + args.output.push('\n'); +} + +struct LineBreak<'a> { + prev: usize, + linebreak: Option<&'a WordInfo<'a>>, + break_before: bool, + demerits: i64, + prev_rat: f32, + length: usize, + fresh: bool, +} + +#[allow(clippy::cognitive_complexity)] +#[allow(clippy::cast_possible_wrap)] +fn find_kp_breakpoints<'a, T: Iterator<Item = &'a WordInfo<'a>>>( + iter: T, + args: &BreakArgs<'a>, +) -> Vec<(&'a WordInfo<'a>, bool)> { + let mut iter = iter.peekable(); + // set up the initial null linebreak + let mut linebreaks = vec![LineBreak { + prev: 0, + linebreak: None, + break_before: false, + demerits: 0, + prev_rat: 0.0, + length: args.init_len, + fresh: false, + }]; + // this vec holds the current active linebreaks; next_ holds the breaks that will be active for + // the next word + let mut active_breaks = vec![0]; + let mut next_active_breaks = vec![]; + + let stretch = args.opts.width - args.opts.goal; + let minlength = args.opts.goal.max(stretch + 1) - stretch; + let mut new_linebreaks = vec![]; + let mut is_sentence_start = false; + let mut least_demerits = 0; + loop { + let Some(w) = iter.next() else { break }; + + // if this is the last word, we don't add additional demerits for this break + let (is_last_word, is_sentence_end) = match iter.peek() { + None => (true, true), + Some(&&WordInfo { + sentence_start: st, + new_line: nl, + .. + }) => (false, st || (nl && w.ends_punct)), + }; + + // should we be adding extra space at the beginning of the next sentence? + let slen = compute_slen(args.uniform, w.new_line, is_sentence_start, false); + + let mut ld_new = i64::MAX; + let mut ld_next = i64::MAX; + let mut ld_idx = 0; + new_linebreaks.clear(); + next_active_breaks.clear(); + // go through each active break, extending it and possibly adding a new active + // break if we are above the minimum required length + #[allow(clippy::explicit_iter_loop)] + for &i in active_breaks.iter() { + let active = &mut linebreaks[i]; + // normalize demerits to avoid overflow, and record if this is the least + active.demerits -= least_demerits; + if active.demerits < ld_next { + ld_next = active.demerits; + ld_idx = i; + } + + // get the new length + let tlen = w.word_nchars + + args.compute_width(w, active.length, active.fresh) + + slen + + active.length; + + // if tlen is longer than args.opts.width, we drop this break from the active list + // otherwise, we extend the break, and possibly add a new break at this point + if tlen <= args.opts.width { + // this break will still be active next time + next_active_breaks.push(i); + // we can put this word on this line + active.fresh = false; + active.length = tlen; + + // if we're above the minlength, we can also consider breaking here + if tlen >= minlength { + let (new_demerits, new_ratio) = if is_last_word { + // there is no penalty for the final line's length + (0, 0.0) + } else { + compute_demerits( + args.opts.goal as isize - tlen as isize, + stretch, + w.word_nchars, + active.prev_rat, + ) + }; + + // do not even consider adding a line that has too many demerits + // also, try to detect overflow by checking signum + let total_demerits = new_demerits + active.demerits; + if new_demerits < BAD_INFTY_SQ + && total_demerits < ld_new + && active.demerits.signum() <= new_demerits.signum() + { + ld_new = total_demerits; + new_linebreaks.push(LineBreak { + prev: i, + linebreak: Some(w), + break_before: false, + demerits: total_demerits, + prev_rat: new_ratio, + length: args.indent_len, + fresh: true, + }); + } + } + } + } + + // if we generated any new linebreaks, add the last one to the list + // the last one is always the best because we don't add to new_linebreaks unless + // it's better than the best one so far + match new_linebreaks.pop() { + None => (), + Some(lb) => { + next_active_breaks.push(linebreaks.len()); + linebreaks.push(lb); + } + } + + if next_active_breaks.is_empty() { + // every potential linebreak is too long! choose the linebreak with the least demerits, ld_idx + let new_break = + restart_active_breaks(args, &linebreaks[ld_idx], ld_idx, w, slen, minlength); + next_active_breaks.push(linebreaks.len()); + linebreaks.push(new_break); + least_demerits = 0; + } else { + // next time around, normalize out the demerits fields + // on active linebreaks to make overflow less likely + least_demerits = cmp::max(ld_next, 0); + } + // swap in new list of active breaks + mem::swap(&mut active_breaks, &mut next_active_breaks); + // If this was the last word in a sentence, the next one must be the first in the next. + is_sentence_start = is_sentence_end; + } + + // return the best path + build_best_path(&linebreaks, &active_breaks) +} + +fn build_best_path<'a>(paths: &[LineBreak<'a>], active: &[usize]) -> Vec<(&'a WordInfo<'a>, bool)> { + // of the active paths, we select the one with the fewest demerits + active + .iter() + .min_by_key(|&&a| paths[a].demerits) + .map(|&(mut best_idx)| { + let mut breakwords = vec![]; + // now, chase the pointers back through the break list, recording + // the words at which we should break + loop { + let next_best = &paths[best_idx]; + match next_best.linebreak { + None => return breakwords, + Some(prev) => { + breakwords.push((prev, next_best.break_before)); + best_idx = next_best.prev; + } + } + } + }) + .unwrap_or_default() +} + +// "infinite" badness is more like (1+BAD_INFTY)^2 because of how demerits are computed +const BAD_INFTY: i64 = 10_000_000; +const BAD_INFTY_SQ: i64 = BAD_INFTY * BAD_INFTY; +// badness = BAD_MULT * abs(r) ^ 3 +const BAD_MULT: f32 = 100.0; +// DR_MULT is multiplier for delta-R between lines +const DR_MULT: f32 = 600.0; +// DL_MULT is penalty multiplier for short words at end of line +const DL_MULT: f32 = 300.0; + +#[allow(clippy::cast_precision_loss)] +#[allow(clippy::cast_possible_truncation)] +fn compute_demerits(delta_len: isize, stretch: usize, wlen: usize, prev_rat: f32) -> (i64, f32) { + // how much stretch are we using? + let ratio = if delta_len == 0 { + 0.0f32 + } else { + delta_len as f32 / stretch as f32 + }; + + // compute badness given the stretch ratio + let bad_linelen = if ratio.abs() > 1.0f32 { + BAD_INFTY + } else { + (BAD_MULT * ratio.powi(3).abs()) as i64 + }; + + // we penalize lines ending in really short words + let bad_wordlen = if wlen >= stretch { + 0 + } else { + (DL_MULT + * ((stretch - wlen) as f32 / (stretch - 1) as f32) + .powi(3) + .abs()) as i64 + }; + + // we penalize lines that have very different ratios from previous lines + let bad_delta_r = (DR_MULT * ((ratio - prev_rat) / 2.0).powi(3).abs()) as i64; + + let demerits = i64::pow(1 + bad_linelen + bad_wordlen + bad_delta_r, 2); + + (demerits, ratio) +} + +#[allow(clippy::cast_possible_wrap)] +fn restart_active_breaks<'a>( + args: &BreakArgs<'a>, + active: &LineBreak<'a>, + act_idx: usize, + w: &'a WordInfo<'a>, + slen: usize, + min: usize, +) -> LineBreak<'a> { + let (break_before, line_length) = if active.fresh { + // never break before a word if that word would be the first on a line + (false, args.indent_len) + } else { + // choose the lesser evil: breaking too early, or breaking too late + let wlen = w.word_nchars + args.compute_width(w, active.length, active.fresh); + let underlen = min as isize - active.length as isize; + let overlen = (wlen + slen + active.length) as isize - args.opts.width as isize; + if overlen > underlen { + // break early, put this word on the next line + (true, args.indent_len + w.word_nchars) + } else { + (false, args.indent_len) + } + }; + + // restart the linebreak. This will be our only active path. + LineBreak { + prev: act_idx, + linebreak: Some(w), + break_before, + demerits: 0, // this is the only active break, so we can reset the demerit count + prev_rat: if break_before { 1.0 } else { -1.0 }, + length: line_length, + fresh: !break_before, + } +} + +// Number of spaces to add before a word, based on mode, newline, sentence start. +#[allow(clippy::fn_params_excessive_bools)] +fn compute_slen(uniform: bool, newline: bool, start: bool, punct: bool) -> usize { + if uniform || newline { + if start || (newline && punct) { 2 } else { 1 } + } else { + 0 + } +} + +// If we're on a fresh line, slen=0 and we slice off leading whitespace. +// Otherwise, compute slen and leave whitespace alone. +#[allow(clippy::fn_params_excessive_bools)] +fn slice_if_fresh( + fresh: bool, + word: &str, + start: usize, + uniform: bool, + newline: bool, + second_start: bool, + punct: bool, +) -> (usize, &str) { + if fresh { + (0, &word[start..]) + } else { + (compute_slen(uniform, newline, second_start, punct), word) + } +} + +// Write a newline and add the indent. +fn write_newline(indent: &str, output: &mut String) { + output.push('\n'); + output.push_str(indent); +} + +// Write the word, along with slen spaces. +fn write_with_spaces(word: &str, slen: usize, output: &mut String) { + if slen == 2 { + output.push_str(" "); + } else if slen == 1 { + output.push(' '); + } + output.push_str(word); +} diff --git a/crates/fmt/src/parasplit.rs b/crates/fmt/src/parasplit.rs new file mode 100644 index 0000000..d4723cb --- /dev/null +++ b/crates/fmt/src/parasplit.rs @@ -0,0 +1,629 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 uutils developers +// SPDX-License-Identifier: MIT +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::iter::Peekable; +use std::slice::Iter; +use unicode_width::UnicodeWidthChar; + +use crate::FmtOptions; + +fn char_width(c: char) -> usize { + if (c as usize) < 0xA0 { + // if it is ASCII, call it exactly 1 wide (including control chars) + // calling control chars' widths 1 is consistent with OpenBSD fmt + 1 + } else { + // otherwise, get the unicode width + // note that we shouldn't actually get None here because only c < 0xA0 + // can return None, but for safety and future-proofing we do it this way + UnicodeWidthChar::width(c).unwrap_or(1) + } +} + +// lines with PSKIP, lacking PREFIX, or which are entirely blank are +// NoFormatLines; otherwise, they are FormatLines +#[derive(Debug)] +pub(super) enum Line { + FormatLine(FileLine), + NoFormatLine(String, bool), +} + +impl Line { + // when we know that it's a FormatLine, as in the ParagraphStream iterator + fn get_formatline(self) -> FileLine { + match self { + Self::FormatLine(fl) => fl, + Self::NoFormatLine(..) => panic!("Found NoFormatLine when expecting FormatLine"), + } + } + + // when we know that it's a NoFormatLine, as in the ParagraphStream iterator + fn get_noformatline(self) -> (String, bool) { + match self { + Self::NoFormatLine(s, b) => (s, b), + Self::FormatLine(..) => panic!("Found FormatLine when expecting NoFormatLine"), + } + } +} + +/// Each line's prefix has to be considered to know whether to merge it with +/// the next line or not +#[derive(Debug)] +pub(super) struct FileLine { + line: String, + /// The end of the indent, always the start of the text + indent_end: usize, + + /// The end of the PREFIX's indent, that is, the spaces before the prefix + prefix_indent_end: usize, + + /// Display length of indent taking into account tabs + indent_len: usize, + + /// PREFIX indent length taking into account tabs + prefix_len: usize, +} + +/// Iterator that produces a stream of Lines from a file +pub(super) struct FileLines<'a> { + opts: &'a FmtOptions, + lines: std::str::Lines<'a>, +} + +impl FileLines<'_> { + fn new<'b>(opts: &'b FmtOptions, lines: std::str::Lines<'b>) -> FileLines<'b> { + FileLines { opts, lines } + } + + /// returns true if this line should be formatted + fn match_prefix(&self, line: &str) -> (bool, usize) { + let Some(prefix) = &self.opts.prefix else { + return (true, 0); + }; + + FileLines::match_prefix_generic(prefix, line, self.opts.exact_prefix) + } + + /// returns true if this line should be formatted + fn match_anti_prefix(&self, line: &str) -> bool { + let Some(anti_prefix) = &self.opts.anti_prefix else { + return true; + }; + + match FileLines::match_prefix_generic(anti_prefix, line, self.opts.exact_anti_prefix) { + (true, _) => false, + (_, _) => true, + } + } + + fn match_prefix_generic(pfx: &str, line: &str, exact: bool) -> (bool, usize) { + if line.starts_with(pfx) { + return (true, 0); + } + + if !exact { + // we do it this way rather than byte indexing to support unicode whitespace chars + for (i, char) in line.char_indices() { + if line[i..].starts_with(pfx) { + return (true, i); + } else if !char.is_whitespace() { + break; + } + } + } + + (false, 0) + } + + fn compute_indent(&self, string: &str, prefix_end: usize) -> (usize, usize, usize) { + let mut prefix_len = 0; + let mut indent_len = 0; + let mut indent_end = 0; + for (os, c) in string.char_indices() { + if os == prefix_end { + // we found the end of the prefix, so this is the printed length of the prefix here + prefix_len = indent_len; + } + + if (os >= prefix_end) && !c.is_whitespace() { + // found first non-whitespace after prefix, this is indent_end + indent_end = os; + break; + } else if c == '\t' { + // compute tab length + indent_len = (indent_len / self.opts.tabwidth + 1) * self.opts.tabwidth; + } else { + // non-tab character + indent_len += char_width(c); + } + } + (indent_end, prefix_len, indent_len) + } +} + +impl Iterator for FileLines<'_> { + type Item = Line; + + fn next(&mut self) -> Option<Line> { + let n = self.lines.next()?; + + // if this line is entirely whitespace, + // emit a blank line + // Err(true) indicates that this was a linebreak, + // which is important to know when detecting mail headers + if n.chars().all(char::is_whitespace) { + return Some(Line::NoFormatLine(String::new(), true)); + } + + let (pmatch, poffset) = self.match_prefix(n); + + // if this line does not match the prefix, + // emit the line unprocessed and iterate again + if !pmatch { + return Some(Line::NoFormatLine(n.to_owned(), false)); + } + + // if the line matches the prefix, but is blank after, + // don't allow lines to be combined through it (that is, + // treat it like a blank line, except that since it's + // not truly blank we will not allow mail headers on the + // following line) + if pmatch + && n[poffset + self.opts.prefix.as_ref().map_or(0, String::len)..] + .chars() + .all(char::is_whitespace) + { + return Some(Line::NoFormatLine(n.to_owned(), false)); + } + + // skip if this line matches the anti_prefix + // (NOTE definition of match_anti_prefix is TRUE if we should process) + if !self.match_anti_prefix(n) { + return Some(Line::NoFormatLine(n.to_owned(), false)); + } + + // figure out the indent, prefix, and prefixindent ending points + let prefix_end = poffset + self.opts.prefix.as_ref().map_or(0, String::len); + let (indent_end, prefix_len, indent_len) = self.compute_indent(n, prefix_end); + + Some(Line::FormatLine(FileLine { + line: n.to_owned(), + indent_end, + prefix_indent_end: poffset, + indent_len, + prefix_len, + })) + } +} + +/// A paragraph : a collection of [`FileLines`] that are to be formatted +/// plus info about the paragraph's indentation +/// +/// We only retain the String from the [`FileLine`]; the other info +/// is only there to help us in deciding how to merge lines into Paragraphs +#[derive(Debug)] +pub(super) struct Paragraph { + /// the lines of the file + lines: Vec<String>, + /// string representing the init, that is, the first line's indent + pub init_str: String, + /// printable length of the init string considering TABWIDTH + pub init_len: usize, + /// byte location of end of init in first line String + init_end: usize, + /// string representing indent + pub indent_str: String, + /// length of above + pub indent_len: usize, + /// byte location of end of indent (in crown and tagged mode, only applies to 2nd line and onward) + indent_end: usize, + /// we need to know if this is a mail header because we do word splitting differently in that case + pub mail_header: bool, +} + +/// An iterator producing a stream of paragraphs from a stream of lines +/// given a set of options. +pub(super) struct ParagraphStream<'a> { + lines: Peekable<FileLines<'a>>, + next_mail: bool, + opts: &'a FmtOptions, +} + +impl ParagraphStream<'_> { + pub(super) fn new<'b>(opts: &'b FmtOptions, text: &'b str) -> ParagraphStream<'b> { + let lines = FileLines::new(opts, text.lines()).peekable(); + // at the beginning of the file, we might find mail headers + ParagraphStream { + lines, + next_mail: true, + opts, + } + } + + /// Detect RFC822 mail header + fn is_mail_header(line: &FileLine) -> bool { + // a mail header begins with either "From " (envelope sender line) + // or with a sequence of printable ASCII chars (33 to 126, inclusive, + // except colon) followed by a colon. + if line.indent_end > 0 { + false + } else { + let l_slice = &line.line[..]; + if l_slice.starts_with("From ") { + true + } else { + let Some(colon_posn) = l_slice.find(':') else { + return false; + }; + + // header field must be nonzero length + if colon_posn == 0 { + return false; + } + + l_slice[..colon_posn] + .chars() + .all(|x| !matches!(x as usize, y if !(33..=126).contains(&y))) + } + } + } +} + +impl Iterator for ParagraphStream<'_> { + type Item = Result<Paragraph, String>; + + #[allow(clippy::cognitive_complexity)] + fn next(&mut self) -> Option<Result<Paragraph, String>> { + // return a NoFormatLine in an Err; it should immediately be output + let noformat = match self.lines.peek()? { + Line::FormatLine(_) => false, + Line::NoFormatLine(_, _) => true, + }; + + // found a NoFormatLine, immediately dump it out + if noformat { + let (s, nm) = self.lines.next().unwrap().get_noformatline(); + self.next_mail = nm; + return Some(Err(s)); + } + + // found a FormatLine, now build a paragraph + let mut init_str = String::new(); + let mut init_end = 0; + let mut init_len = 0; + let mut indent_str = String::new(); + let mut indent_end = 0; + let mut indent_len = 0; + let mut prefix_len = 0; + let mut prefix_indent_end = 0; + let mut p_lines = Vec::new(); + + let mut in_mail = false; + let mut second_done = false; // for when we use crown or tagged mode + loop { + // peek ahead + // need to explicitly force fl out of scope before we can call self.lines.next() + let Some(Line::FormatLine(fl)) = self.lines.peek() else { + break; + }; + + if p_lines.is_empty() { + // first time through the loop, get things set up + // detect mail header + if self.opts.mail && self.next_mail && ParagraphStream::is_mail_header(fl) { + in_mail = true; + // there can't be any indent or prefixindent because otherwise is_mail_header + // would fail since there cannot be any whitespace before the colon in a + // valid header field + indent_str.push_str(" "); + indent_len = 2; + } else { + if self.opts.crown_margin || self.opts.tagged_paragraph { + init_str.push_str(&fl.line[..fl.indent_end]); + init_len = fl.indent_len; + init_end = fl.indent_end; + } else { + second_done = true; + } + + // these will be overwritten in the 2nd line of crown or tagged mode, but + // we are not guaranteed to get to the 2nd line, e.g., if the next line + // is a NoFormatLine or None. Thus, we set sane defaults the 1st time around + indent_str.push_str(&fl.line[..fl.indent_end]); + indent_len = fl.indent_len; + indent_end = fl.indent_end; + + // save these to check for matching lines + prefix_len = fl.prefix_len; + prefix_indent_end = fl.prefix_indent_end; + + // in tagged mode, add 4 spaces of additional indenting by default + // (gnu fmt's behavior is different: it seems to find the closest column to + // indent_end that is divisible by 3. But honestly that behavior seems + // pretty arbitrary. + // Perhaps a better default would be 1 TABWIDTH? But ugh that's so big. + if self.opts.tagged_paragraph { + indent_str.push_str(" "); + indent_len += 4; + } + } + } else if in_mail { + // lines following mail headers must begin with spaces + if fl.indent_end == 0 || (self.opts.prefix.is_some() && fl.prefix_indent_end == 0) { + break; // this line does not begin with spaces + } + } else if !second_done { + // now we have enough info to handle crown margin and tagged mode + + // in both crown and tagged modes we require that prefix_len is the same + if prefix_len != fl.prefix_len || prefix_indent_end != fl.prefix_indent_end { + break; + } + + // in tagged mode, indent has to be *different* on following lines + if self.opts.tagged_paragraph + && indent_len - 4 == fl.indent_len + && indent_end == fl.indent_end + { + break; + } + + // this is part of the same paragraph, get the indent info from this line + indent_str.clear(); + indent_str.push_str(&fl.line[..fl.indent_end]); + indent_len = fl.indent_len; + indent_end = fl.indent_end; + + second_done = true; + } else { + // detect mismatch + if indent_end != fl.indent_end + || prefix_indent_end != fl.prefix_indent_end + || indent_len != fl.indent_len + || prefix_len != fl.prefix_len + { + break; + } + } + + p_lines.push(self.lines.next().unwrap().get_formatline().line); + + // when we're in split-only mode, we never join lines, so stop here + if self.opts.split_only { + break; + } + } + + // if this was a mail header, then the next line can be detected as one. Otherwise, it cannot. + // NOTE next_mail is true at ParagraphStream instantiation, and is set to true after a blank + // NoFormatLine. + self.next_mail = in_mail; + + Some(Ok(Paragraph { + lines: p_lines, + init_str, + init_len, + init_end, + indent_str, + indent_len, + indent_end, + mail_header: in_mail, + })) + } +} + +pub(super) struct ParaWords<'a> { + opts: &'a FmtOptions, + para: &'a Paragraph, + words: Vec<WordInfo<'a>>, +} + +impl<'a> ParaWords<'a> { + pub(super) fn new(opts: &'a FmtOptions, para: &'a Paragraph) -> Self { + let mut pw = ParaWords { + opts, + para, + words: Vec::new(), + }; + pw.create_words(); + pw + } + + fn create_words(&mut self) { + if self.para.mail_header { + // no extra spacing for mail headers; always exactly 1 space + // safe to trim_start on every line of a mail header, since the + // first line is guaranteed not to have any spaces + self.words.extend( + self.para + .lines + .iter() + .flat_map(|x| x.split_whitespace()) + .map(|x| WordInfo { + word: x, + word_start: 0, + word_nchars: x.len(), // OK for mail headers; only ASCII allowed (unicode is escaped) + before_tab: None, + after_tab: 0, + sentence_start: false, + ends_punct: false, + new_line: false, + }), + ); + } else { + // first line + self.words + .extend(if self.opts.crown_margin || self.opts.tagged_paragraph { + // crown and tagged mode has the "init" in the first line, so slice from there + WordSplit::new(self.opts, &self.para.lines[0][self.para.init_end..]) + } else { + // otherwise we slice from the indent + WordSplit::new(self.opts, &self.para.lines[0][self.para.indent_end..]) + }); + + if self.para.lines.len() > 1 { + let indent_end = self.para.indent_end; + let opts = self.opts; + self.words.extend( + self.para + .lines + .iter() + .skip(1) + .flat_map(|x| WordSplit::new(opts, &x[indent_end..])), + ); + } + } + } + + pub(super) fn words(&'a self) -> Iter<'a, WordInfo<'a>> { + self.words.iter() + } +} + +struct WordSplit<'a> { + opts: &'a FmtOptions, + string: &'a str, + length: usize, + position: usize, + prev_punct: bool, +} + +impl WordSplit<'_> { + fn analyze_tabs(&self, string: &str) -> (Option<usize>, usize, Option<usize>) { + // given a string, determine (length before tab) and (printed length after first tab) + // if there are no tabs, beforetab = -1 and aftertab is the printed length + let mut beforetab = None; + let mut aftertab = 0; + let mut word_start = None; + for (os, c) in string.char_indices() { + if !c.is_whitespace() { + word_start = Some(os); + break; + } else if c == '\t' { + if beforetab.is_none() { + beforetab = Some(aftertab); + aftertab = 0; + } else { + aftertab = (aftertab / self.opts.tabwidth + 1) * self.opts.tabwidth; + } + } else { + aftertab += 1; + } + } + (beforetab, aftertab, word_start) + } +} + +impl WordSplit<'_> { + fn new<'b>(opts: &'b FmtOptions, string: &'b str) -> WordSplit<'b> { + // wordsplits *must* start at a non-whitespace character + let trim_string = string.trim_start(); + WordSplit { + opts, + string: trim_string, + length: string.len(), + position: 0, + prev_punct: false, + } + } + + fn is_punctuation(c: char) -> bool { + matches!(c, '!' | '.' | '?') + } +} + +pub(super) struct WordInfo<'a> { + pub word: &'a str, + pub word_start: usize, + pub word_nchars: usize, + pub before_tab: Option<usize>, + pub after_tab: usize, + pub sentence_start: bool, + pub ends_punct: bool, + pub new_line: bool, +} + +// returns (&str, is_start_of_sentence) +impl<'a> Iterator for WordSplit<'a> { + type Item = WordInfo<'a>; + + fn next(&mut self) -> Option<WordInfo<'a>> { + if self.position >= self.length { + return None; + } + + let old_position = self.position; + let new_line = old_position == 0; + + // find the start of the next word, and record if we find a tab character + let (before_tab, after_tab, word_start) = + if let (b, a, Some(s)) = self.analyze_tabs(&self.string[old_position..]) { + (b, a, s + old_position) + } else { + self.position = self.length; + return None; + }; + + // find the beginning of the next whitespace + // note that this preserves the invariant that self.position + // points to whitespace character OR end of string + let mut word_nchars = 0; + self.position = match self.string[word_start..].find(|x: char| { + if x.is_whitespace() { + true + } else { + word_nchars += char_width(x); + false + } + }) { + None => self.length, + Some(s) => s + word_start, + }; + + let word_start_relative = word_start - old_position; + // if the previous sentence was punctuation and this sentence has >2 whitespace or one tab, is a new sentence. + let is_start_of_sentence = + self.prev_punct && (before_tab.is_some() || word_start_relative > 1); + + // now record whether this word ends in punctuation + self.prev_punct = match self.string[..self.position].chars().next_back() { + Some(ch) => WordSplit::is_punctuation(ch), + _ => panic!("fatal: expected word not to be empty"), + }; + + let (word, word_start_relative, before_tab, after_tab) = if self.opts.uniform { + (&self.string[word_start..self.position], 0, None, 0) + } else { + ( + &self.string[old_position..self.position], + word_start_relative, + before_tab, + after_tab, + ) + }; + + Some(WordInfo { + word, + word_start: word_start_relative, + word_nchars, + before_tab, + after_tab, + sentence_start: is_start_of_sentence, + ends_punct: self.prev_punct, + new_line, + }) + } +} diff --git a/crates/libmpv2/CHANGELOG.md b/crates/libmpv2/CHANGELOG.md index dc6f861..a3d14d7 100644 --- a/crates/libmpv2/CHANGELOG.md +++ b/crates/libmpv2/CHANGELOG.md @@ -16,7 +16,7 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. ## Version 3.0.0 -- \[breaking\] Support libmpv version 2.0 (mpv version 0.35.0). Mpv versions \<= +- [breaking] Support libmpv version 2.0 (mpv version 0.35.0). Mpv versions \<= 0.34.0 will no longer be supported. - Add OpenGL rendering @@ -29,10 +29,10 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. ## Version 2.0.0 - Add method `Mpv::with_initializer` to set options before initialization -- \[breaking\] Borrow `&mut self` in `wait_event` to disallow using two events +- [breaking] Borrow `&mut self` in `wait_event` to disallow using two events where the first points to data freed in the second `wait_event` call -- \[breaking\] `PropertyData<'_>` is no longer `Clone` or `PartialEq`, - `Event<'_>` is no longer `Clone` to avoid cloning/comparing `MpvNode` +- [breaking] `PropertyData<'_>` is no longer `Clone` or `PartialEq`, `Event<'_>` + is no longer `Clone` to avoid cloning/comparing `MpvNode` ## Version 1.1.0 diff --git a/crates/libmpv2/Cargo.toml b/crates/libmpv2/Cargo.toml index a8a4ed6..67fbfec 100644 --- a/crates/libmpv2/Cargo.toml +++ b/crates/libmpv2/Cargo.toml @@ -24,12 +24,11 @@ publish = false [dependencies] libmpv2-sys = { path = "libmpv2-sys" } -thiserror = "2.0.7" log.workspace = true [dev-dependencies] crossbeam = "0.8" -sdl2 = "0.37.0" +sdl2 = "0.38.0" [features] default = ["protocols", "render"] diff --git a/crates/libmpv2/examples/events.rs b/crates/libmpv2/examples/events.rs index 8f7c79f..e502d5c 100644 --- a/crates/libmpv2/examples/events.rs +++ b/crates/libmpv2/examples/events.rs @@ -45,25 +45,27 @@ fn main() -> Result<()> { // Trigger `Event::EndFile`. mpv.command("playlist-next", &["force"]).unwrap(); }); - scope.spawn(move |_| loop { - let ev = ev_ctx.wait_event(600.).unwrap_or(Err(Error::Null)); + scope.spawn(move |_| { + loop { + let ev = ev_ctx.wait_event(600.).unwrap_or(Err(Error::Null)); - match ev { - Ok(Event::EndFile(r)) => { - println!("Exiting! Reason: {:?}", r); - break; - } + match ev { + Ok(Event::EndFile(r)) => { + println!("Exiting! Reason: {:?}", r); + break; + } - Ok(Event::PropertyChange { - name: "demuxer-cache-state", - change: PropertyData::Node(mpv_node), - .. - }) => { - let ranges = seekable_ranges(mpv_node); - println!("Seekable ranges updated: {:?}", ranges); + Ok(Event::PropertyChange { + name: "demuxer-cache-state", + change: PropertyData::Node(mpv_node), + .. + }) => { + let ranges = seekable_ranges(mpv_node); + println!("Seekable ranges updated: {:?}", ranges); + } + Ok(e) => println!("Event triggered: {:?}", e), + Err(e) => println!("Event errored: {:?}", e), } - Ok(e) => println!("Event triggered: {:?}", e), - Err(e) => println!("Event errored: {:?}", e), } }); }) diff --git a/crates/libmpv2/examples/opengl.rs b/crates/libmpv2/examples/opengl.rs index 1de307f..9f595aa 100644 --- a/crates/libmpv2/examples/opengl.rs +++ b/crates/libmpv2/examples/opengl.rs @@ -9,8 +9,8 @@ // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. use libmpv2::{ - render::{OpenGLInitParams, RenderContext, RenderParam, RenderParamApiType}, Mpv, + render::{OpenGLInitParams, RenderContext, RenderParam, RenderParamApiType}, }; use std::{env, ffi::c_void}; diff --git a/crates/libmpv2/libmpv2-sys/Cargo.toml b/crates/libmpv2/libmpv2-sys/Cargo.toml index b0514b8..96141d3 100644 --- a/crates/libmpv2/libmpv2-sys/Cargo.toml +++ b/crates/libmpv2/libmpv2-sys/Cargo.toml @@ -23,4 +23,4 @@ rust-version.workspace = true publish = false [build-dependencies] -bindgen = { version = "0.71.1" } +bindgen = { version = "0.72.0" } diff --git a/crates/libmpv2/libmpv2-sys/build.rs b/crates/libmpv2/libmpv2-sys/build.rs index bf9a02e..45c2450 100644 --- a/crates/libmpv2/libmpv2-sys/build.rs +++ b/crates/libmpv2/libmpv2-sys/build.rs @@ -30,7 +30,9 @@ fn main() { ), "--verbose", ]) - .generate_comments(true) + // NOTE(@bpeetz): The comments are interpreted as doc-tests, + // which obviously fail, as the code is c. <2025-06-16> + .generate_comments(false) .generate() .expect("Unable to generate bindings"); diff --git a/crates/libmpv2/src/lib.rs b/crates/libmpv2/src/lib.rs index 4d8d18a..f6c2103 100644 --- a/crates/libmpv2/src/lib.rs +++ b/crates/libmpv2/src/lib.rs @@ -35,7 +35,7 @@ use std::os::raw as ctype; pub const MPV_CLIENT_API_MAJOR: ctype::c_ulong = 2; pub const MPV_CLIENT_API_MINOR: ctype::c_ulong = 2; pub const MPV_CLIENT_API_VERSION: ctype::c_ulong = - MPV_CLIENT_API_MAJOR << 16 | MPV_CLIENT_API_MINOR; + (MPV_CLIENT_API_MAJOR << 16) | MPV_CLIENT_API_MINOR; mod mpv; #[cfg(test)] @@ -67,8 +67,8 @@ pub mod mpv_error { pub use libmpv2_sys::mpv_error_MPV_ERROR_INVALID_PARAMETER as InvalidParameter; pub use libmpv2_sys::mpv_error_MPV_ERROR_LOADING_FAILED as LoadingFailed; pub use libmpv2_sys::mpv_error_MPV_ERROR_NOMEM as NoMem; - pub use libmpv2_sys::mpv_error_MPV_ERROR_NOTHING_TO_PLAY as NothingToPlay; pub use libmpv2_sys::mpv_error_MPV_ERROR_NOT_IMPLEMENTED as NotImplemented; + pub use libmpv2_sys::mpv_error_MPV_ERROR_NOTHING_TO_PLAY as NothingToPlay; pub use libmpv2_sys::mpv_error_MPV_ERROR_OPTION_ERROR as OptionError; pub use libmpv2_sys::mpv_error_MPV_ERROR_OPTION_FORMAT as OptionFormat; pub use libmpv2_sys::mpv_error_MPV_ERROR_OPTION_NOT_FOUND as OptionNotFound; diff --git a/crates/libmpv2/src/mpv.rs b/crates/libmpv2/src/mpv.rs index 07d0976..d8164c0 100644 --- a/crates/libmpv2/src/mpv.rs +++ b/crates/libmpv2/src/mpv.rs @@ -184,7 +184,7 @@ pub mod mpv_node { pub mod sys_node { use super::{DropWrapper, MpvNode, MpvNodeArrayIter, MpvNodeMapIter}; - use crate::{mpv_error, mpv_format, Error, Result}; + use crate::{Error, Result, mpv_error, mpv_format}; use std::rc::Rc; #[derive(Debug, Clone)] @@ -375,14 +375,14 @@ unsafe impl SetData for String { /// Wrapper around an `&str` returned by mpv, that properly deallocates it with mpv's allocator. #[derive(Debug, Hash, Eq, PartialEq)] pub struct MpvStr<'a>(&'a str); -impl<'a> Deref for MpvStr<'a> { +impl Deref for MpvStr<'_> { type Target = str; fn deref(&self) -> &str { self.0 } } -impl<'a> Drop for MpvStr<'a> { +impl Drop for MpvStr<'_> { fn drop(&mut self) { unsafe { libmpv2_sys::mpv_free(self.0.as_ptr() as *mut u8 as _) }; } @@ -403,7 +403,7 @@ unsafe impl<'a> GetData for MpvStr<'a> { } } -unsafe impl<'a> SetData for &'a str { +unsafe impl SetData for &str { fn call_as_c_void<T, F: FnMut(*mut ctype::c_void) -> Result<T>>(self, mut fun: F) -> Result<T> { let string = CString::new(self)?; fun((&mut string.as_ptr()) as *mut *const ctype::c_char as *mut _) @@ -511,9 +511,8 @@ impl Mpv { } initializer(MpvInitializer { ctx })?; - mpv_err((), unsafe { libmpv2_sys::mpv_initialize(ctx) }).map_err(|err| { + mpv_err((), unsafe { libmpv2_sys::mpv_initialize(ctx) }).inspect_err(|_| { unsafe { libmpv2_sys::mpv_terminate_destroy(ctx) }; - err })?; let ctx = unsafe { NonNull::new_unchecked(ctx) }; @@ -526,19 +525,6 @@ impl Mpv { }) } - /// Execute a command - pub fn execute(&self, name: &str, args: &[&str]) -> Result<()> { - if args.is_empty() { - debug!("Running mpv command: '{}'", name); - } else { - debug!("Running mpv command: '{} {}'", name, args.join(" ")); - } - - self.command(name, args)?; - - Ok(()) - } - /// Load a configuration file. The path has to be absolute, and a file. pub fn load_config(&self, path: &str) -> Result<()> { let file = CString::new(path)?.into_raw(); @@ -562,33 +548,40 @@ impl Mpv { /// Send a command to the `Mpv` instance. This uses `mpv_command_string` internally, /// so that the syntax is the same as described in the [manual for the input.conf](https://mpv.io/manual/master/#list-of-input-commands). /// - /// Note that you may have to escape strings with `""` when they contain spaces. + /// Note that this function escapes the arguments for you. /// /// # Examples /// - /// ```dont_run - /// # use libmpv2::{Mpv}; - /// # use libmpv2::mpv_node::MpvNode; - /// # use libmpv2::mpv::errors::Result; - /// # use std::collections::HashMap; - /// # - /// # fn main() -> Result<()> { - /// # let mpv = Mpv::new()?; + /// ```text + ///# use libmpv2::{Mpv}; + ///# use libmpv2::mpv_node::MpvNode; + ///# use libmpv2::mpv::errors::Result; + ///# use std::collections::HashMap; + ///# + ///# fn main() -> Result<()> { + ///# let mpv = Mpv::new()?; /// mpv.command("loadfile", &["test-data/jellyfish.mp4", "append-play"]).unwrap(); - /// # let node = mpv.get_property::<MpvNode>("playlist").unwrap(); - /// # let mut list = node.array().unwrap().collect::<Vec<_>>(); - /// # let map = list.pop().unwrap().map().unwrap().collect::<HashMap<_, _>>(); - /// # assert_eq!(map, HashMap::from([(String::from("id"), MpvNode::Int64(1)), (String::from("current"), MpvNode::Flag(true)), (String::from("filename"), MpvNode::String(String::from("test-data/jellyfish.mp4")))])); - /// # Ok(()) - /// # } + ///# let node = mpv.get_property::<MpvNode>("playlist").unwrap(); + ///# let mut list = node.array().unwrap().collect::<Vec<_>>(); + ///# let map = list.pop().unwrap().map().unwrap().collect::<HashMap<_, _>>(); + ///# assert_eq!(map, HashMap::from([(String::from("id"), MpvNode::Int64(1)), (String::from("current"), MpvNode::Flag(true)), (String::from("filename"), MpvNode::String(String::from("test-data/jellyfish.mp4")))])); + ///# Ok(()) + ///# } /// ``` pub fn command(&self, name: &str, args: &[&str]) -> Result<()> { - let mut cmd = name.to_owned(); + fn escape(input: &str) -> String { + input.replace('"', "\\\"") + } + + let mut cmd = escape(name); for elem in args { cmd.push(' '); - cmd.push_str(elem); + cmd.push('"'); + cmd.push_str(&escape(elem)); + cmd.push('"'); } + debug!("Running mpv command: '{}'", cmd); let raw = CString::new(cmd)?; mpv_err((), unsafe { @@ -597,7 +590,9 @@ impl Mpv { } /// Set the value of a property. - pub fn set_property<T: SetData>(&self, name: &str, data: T) -> Result<()> { + pub fn set_property<T: SetData + std::fmt::Display>(&self, name: &str, data: T) -> Result<()> { + debug!("Setting mpv property: '{name}' = '{data}'"); + let name = CString::new(name)?; let format = T::get_format().as_mpv_format() as _; data.call_as_c_void(|ptr| { diff --git a/crates/libmpv2/src/mpv/errors.rs b/crates/libmpv2/src/mpv/errors.rs index a2baee5..a2d3dd8 100644 --- a/crates/libmpv2/src/mpv/errors.rs +++ b/crates/libmpv2/src/mpv/errors.rs @@ -8,36 +8,52 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use std::{ffi::NulError, os::raw as ctype, str::Utf8Error}; - -use thiserror::Error; +use std::{ffi::NulError, fmt::Display, os::raw as ctype, str::Utf8Error}; use super::mpv_error; #[allow(missing_docs)] pub type Result<T> = ::std::result::Result<T, Error>; -#[derive(Error, Debug)] +#[derive(Debug)] pub enum Error { - #[error("loading file failed: {error}")] - Loadfile { error: String }, + Loadfile { + error: String, + }, - #[error("version mismatch detected! Linked version ({linked}) is unequal to the loaded version ({loaded})")] VersionMismatch { linked: ctype::c_ulong, loaded: ctype::c_ulong, }, - #[error("invalid utf8 returned")] InvalidUtf8, - #[error("null pointer returned")] Null, - #[error("raw error returned: {}", to_string_mpv_error(*(.0)))] Raw(crate::MpvError), } +impl std::error::Error for Error {} + +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Loadfile { error } => write!(f, "loading file failed: {error}"), + Error::VersionMismatch { linked, loaded } => write!( + f, + "version mismatch detected! Linked version ({linked}) is unequal to the loaded version ({loaded})" + ), + Error::InvalidUtf8 => f.write_str("invalid utf8 returned"), + Error::Null => f.write_str("null pointer returned"), + Error::Raw(raw) => write!( + f, + include_str!("./raw_error_warning.txt"), + to_string_mpv_error(*(raw)) + ), + } + } +} + impl From<NulError> for Error { fn from(_other: NulError) -> Error { Error::Null @@ -76,35 +92,70 @@ fn to_string_mpv_error_raw(num: crate::MpvError) -> (&'static str, &'static str) mpv_error::NoMem => ("Memory allocation failed.", ""), - mpv_error::Uninitialized => ("The mpv core wasn't configured and initialized yet", " See the notes in mpv_create()."), + mpv_error::Uninitialized => ( + "The mpv core wasn't configured and initialized yet", + " See the notes in mpv_create().", + ), - mpv_error::InvalidParameter => ("Generic catch-all error if a parameter is set to an invalid or unsupported value.", "This is used if there is no better error code."), + mpv_error::InvalidParameter => ( + "Generic catch-all error if a parameter is set to an invalid or unsupported value.", + "This is used if there is no better error code.", + ), mpv_error::OptionNotFound => ("Trying to set an option that doesn't exist.", ""), - mpv_error::OptionFormat => ("Trying to set an option using an unsupported MPV_FORMAT.", ""), - mpv_error::OptionError => ("Setting the option failed", " Typically this happens if the provided option value could not be parsed."), + mpv_error::OptionFormat => ( + "Trying to set an option using an unsupported MPV_FORMAT.", + "", + ), + mpv_error::OptionError => ( + "Setting the option failed", + " Typically this happens if the provided option value could not be parsed.", + ), mpv_error::PropertyNotFound => ("The accessed property doesn't exist.", ""), - mpv_error::PropertyFormat => ("Trying to set or get a property using an unsupported MPV_FORMAT.", ""), - mpv_error::PropertyUnavailable => ("The property exists, but is not available", "This usually happens when the associated subsystem is not active, e.g. querying audio parameters while audio is disabled."), + mpv_error::PropertyFormat => ( + "Trying to set or get a property using an unsupported MPV_FORMAT.", + "", + ), + mpv_error::PropertyUnavailable => ( + "The property exists, but is not available", + "This usually happens when the associated subsystem is not active, e.g. querying audio parameters while audio is disabled.", + ), mpv_error::PropertyError => ("Error setting or getting a property.", ""), - mpv_error::Command => ("General error when running a command with mpv_command and similar.", ""), + mpv_error::Command => ( + "General error when running a command with mpv_command and similar.", + "", + ), - mpv_error::LoadingFailed => ("Generic error on loading (usually used with mpv_event_end_file.error).", ""), + mpv_error::LoadingFailed => ( + "Generic error on loading (usually used with mpv_event_end_file.error).", + "", + ), mpv_error::AoInitFailed => ("Initializing the audio output failed.", ""), mpv_error::VoInitFailed => ("Initializing the video output failed.", ""), - mpv_error::NothingToPlay => ("There was no audio or video data to play", "This also happens if the file was recognized, but did not contain any audio or video streams, or no streams were selected."), + mpv_error::NothingToPlay => ( + "There was no audio or video data to play", + "This also happens if the file was recognized, but did not contain any audio or video streams, or no streams were selected.", + ), - mpv_error::UnknownFormat => (" * When trying to load the file, the file format could not be determined, or the file was too broken to open it.", ""), + mpv_error::UnknownFormat => ( + " * When trying to load the file, the file format could not be determined, or the file was too broken to open it.", + "", + ), - mpv_error::Generic => ("Generic error for signaling that certain system requirements are not fulfilled.", ""), + mpv_error::Generic => ( + "Generic error for signaling that certain system requirements are not fulfilled.", + "", + ), mpv_error::NotImplemented => ("The API function which was called is a stub only", ""), mpv_error::Unsupported => ("Unspecified error.", ""), - mpv_error::Success => unreachable!("This is not an error. It's just here, to ensure that the 0 case marks an success'"), + mpv_error::Success => unreachable!( + "This is not an error. It's just here, to ensure that the 0 case marks an success'" + ), _ => unreachable!("Mpv seems to have changed it's constants."), } } diff --git a/crates/libmpv2/src/mpv/events.rs b/crates/libmpv2/src/mpv/events.rs index 6fb4683..f10ff6e 100644 --- a/crates/libmpv2/src/mpv/events.rs +++ b/crates/libmpv2/src/mpv/events.rs @@ -11,7 +11,7 @@ use crate::mpv_node::sys_node::SysMpvNode; use crate::{mpv::mpv_err, *}; -use std::ffi::{c_void, CString}; +use std::ffi::{CString, c_void}; use std::os::raw as ctype; use std::ptr::NonNull; use std::slice; @@ -70,26 +70,28 @@ impl<'a> PropertyData<'a> { // SAFETY: meant to extract the data from an event property. See `mpv_event_property` in // `client.h` unsafe fn from_raw(format: MpvFormat, ptr: *mut ctype::c_void) -> Result<PropertyData<'a>> { - assert!(!ptr.is_null()); - match format { - mpv_format::Flag => Ok(PropertyData::Flag(*(ptr as *mut bool))), - mpv_format::String => { - let char_ptr = *(ptr as *mut *mut ctype::c_char); - Ok(PropertyData::Str(mpv_cstr_to_str!(char_ptr)?)) - } - mpv_format::OsdString => { - let char_ptr = *(ptr as *mut *mut ctype::c_char); - Ok(PropertyData::OsdStr(mpv_cstr_to_str!(char_ptr)?)) - } - mpv_format::Double => Ok(PropertyData::Double(*(ptr as *mut f64))), - mpv_format::Int64 => Ok(PropertyData::Int64(*(ptr as *mut i64))), - mpv_format::Node => { - let sys_node = *(ptr as *mut libmpv2_sys::mpv_node); - let node = SysMpvNode::new(sys_node, false); - return Ok(PropertyData::Node(node.value().unwrap())); + unsafe { + assert!(!ptr.is_null()); + match format { + mpv_format::Flag => Ok(PropertyData::Flag(*(ptr as *mut bool))), + mpv_format::String => { + let char_ptr = *(ptr as *mut *mut ctype::c_char); + Ok(PropertyData::Str(mpv_cstr_to_str!(char_ptr)?)) + } + mpv_format::OsdString => { + let char_ptr = *(ptr as *mut *mut ctype::c_char); + Ok(PropertyData::OsdStr(mpv_cstr_to_str!(char_ptr)?)) + } + mpv_format::Double => Ok(PropertyData::Double(*(ptr as *mut f64))), + mpv_format::Int64 => Ok(PropertyData::Int64(*(ptr as *mut i64))), + mpv_format::Node => { + let sys_node = *(ptr as *mut libmpv2_sys::mpv_node); + let node = SysMpvNode::new(sys_node, false); + Ok(PropertyData::Node(node.value().unwrap())) + } + mpv_format::None => unreachable!(), + _ => unimplemented!(), } - mpv_format::None => unreachable!(), - _ => unimplemented!(), } } } @@ -146,11 +148,13 @@ pub enum Event<'a> { } unsafe extern "C" fn wu_wrapper<F: Fn() + Send + 'static>(ctx: *mut c_void) { - if ctx.is_null() { - panic!("ctx for wakeup wrapper is NULL"); - } + unsafe { + if ctx.is_null() { + panic!("ctx for wakeup wrapper is NULL"); + } - (*(ctx as *mut F))(); + (*(ctx as *mut F))(); + } } /// Context to listen to events. diff --git a/crates/libmpv2/src/mpv/protocol.rs b/crates/libmpv2/src/mpv/protocol.rs index 31a5933..ee33411 100644 --- a/crates/libmpv2/src/mpv/protocol.rs +++ b/crates/libmpv2/src/mpv/protocol.rs @@ -17,7 +17,7 @@ use std::os::raw as ctype; use std::panic; use std::panic::RefUnwindSafe; use std::slice; -use std::sync::{atomic::Ordering, Mutex}; +use std::sync::{Mutex, atomic::Ordering}; impl Mpv { /// Create a context with which custom protocols can be registered. @@ -63,26 +63,28 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = user_data as *mut ProtocolData<T, U>; + unsafe { + let data = user_data as *mut ProtocolData<T, U>; - (*info).cookie = user_data; - (*info).read_fn = Some(read_wrapper::<T, U>); - (*info).seek_fn = Some(seek_wrapper::<T, U>); - (*info).size_fn = Some(size_wrapper::<T, U>); - (*info).close_fn = Some(close_wrapper::<T, U>); + (*info).cookie = user_data; + (*info).read_fn = Some(read_wrapper::<T, U>); + (*info).seek_fn = Some(seek_wrapper::<T, U>); + (*info).size_fn = Some(size_wrapper::<T, U>); + (*info).close_fn = Some(close_wrapper::<T, U>); - let ret = panic::catch_unwind(|| { - let uri = mpv_cstr_to_str!(uri as *const _).unwrap(); - ptr::write( - (*data).cookie, - ((*data).open_fn)(&mut (*data).user_data, uri), - ); - }); + let ret = panic::catch_unwind(|| { + let uri = mpv_cstr_to_str!(uri as *const _).unwrap(); + ptr::write( + (*data).cookie, + ((*data).open_fn)(&mut (*data).user_data, uri), + ); + }); - if ret.is_ok() { - 0 - } else { - mpv_error::Generic as _ + if ret.is_ok() { + 0 + } else { + mpv_error::Generic as _ + } } } @@ -95,16 +97,14 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = cookie as *mut ProtocolData<T, U>; + unsafe { + let data = cookie as *mut ProtocolData<T, U>; - let ret = panic::catch_unwind(|| { - let slice = slice::from_raw_parts_mut(buf, nbytes as _); - ((*data).read_fn)(&mut *(*data).cookie, slice) - }); - if let Ok(ret) = ret { - ret - } else { - -1 + let ret = panic::catch_unwind(|| { + let slice = slice::from_raw_parts_mut(buf, nbytes as _); + ((*data).read_fn)(&mut *(*data).cookie, slice) + }); + ret.unwrap_or(-1) } } @@ -113,18 +113,21 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = cookie as *mut ProtocolData<T, U>; + unsafe { + let data = cookie as *mut ProtocolData<T, U>; - if (*data).seek_fn.is_none() { - return mpv_error::Unsupported as _; - } + if (*data).seek_fn.is_none() { + return mpv_error::Unsupported as _; + } - let ret = - panic::catch_unwind(|| (*(*data).seek_fn.as_ref().unwrap())(&mut *(*data).cookie, offset)); - if let Ok(ret) = ret { - ret - } else { - mpv_error::Generic as _ + let ret = panic::catch_unwind(|| { + (*(*data).seek_fn.as_ref().unwrap())(&mut *(*data).cookie, offset) + }); + if let Ok(ret) = ret { + ret + } else { + mpv_error::Generic as _ + } } } @@ -133,17 +136,20 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = cookie as *mut ProtocolData<T, U>; + unsafe { + let data = cookie as *mut ProtocolData<T, U>; - if (*data).size_fn.is_none() { - return mpv_error::Unsupported as _; - } + if (*data).size_fn.is_none() { + return mpv_error::Unsupported as _; + } - let ret = panic::catch_unwind(|| (*(*data).size_fn.as_ref().unwrap())(&mut *(*data).cookie)); - if let Ok(ret) = ret { - ret - } else { - mpv_error::Unsupported as _ + let ret = + panic::catch_unwind(|| (*(*data).size_fn.as_ref().unwrap())(&mut *(*data).cookie)); + if let Ok(ret) = ret { + ret + } else { + mpv_error::Unsupported as _ + } } } @@ -153,9 +159,11 @@ where T: RefUnwindSafe, U: RefUnwindSafe, { - let data = Box::from_raw(cookie as *mut ProtocolData<T, U>); + unsafe { + let data = Box::from_raw(cookie as *mut ProtocolData<T, U>); - panic::catch_unwind(|| (data.close_fn)(Box::from_raw(data.cookie))); + panic::catch_unwind(|| (data.close_fn)(Box::from_raw(data.cookie))); + } } struct ProtocolData<T, U> { @@ -177,8 +185,8 @@ pub struct ProtocolContext<'parent, T: RefUnwindSafe, U: RefUnwindSafe> { _does_not_outlive: PhantomData<&'parent Mpv>, } -unsafe impl<'parent, T: RefUnwindSafe, U: RefUnwindSafe> Send for ProtocolContext<'parent, T, U> {} -unsafe impl<'parent, T: RefUnwindSafe, U: RefUnwindSafe> Sync for ProtocolContext<'parent, T, U> {} +unsafe impl<T: RefUnwindSafe, U: RefUnwindSafe> Send for ProtocolContext<'_, T, U> {} +unsafe impl<T: RefUnwindSafe, U: RefUnwindSafe> Sync for ProtocolContext<'_, T, U> {} impl<'parent, T: RefUnwindSafe, U: RefUnwindSafe> ProtocolContext<'parent, T, U> { fn new( @@ -228,20 +236,23 @@ impl<T: RefUnwindSafe, U: RefUnwindSafe> Protocol<T, U> { seek_fn: Option<StreamSeek<T>>, size_fn: Option<StreamSize<T>>, ) -> Protocol<T, U> { - let c_layout = Layout::from_size_align(mem::size_of::<T>(), mem::align_of::<T>()).unwrap(); - let cookie = alloc::alloc(c_layout) as *mut T; - let data = Box::into_raw(Box::new(ProtocolData { - cookie, - user_data, + unsafe { + let c_layout = + Layout::from_size_align(mem::size_of::<T>(), mem::align_of::<T>()).unwrap(); + let cookie = alloc::alloc(c_layout) as *mut T; + let data = Box::into_raw(Box::new(ProtocolData { + cookie, + user_data, - open_fn, - close_fn, - read_fn, - seek_fn, - size_fn, - })); + open_fn, + close_fn, + read_fn, + seek_fn, + size_fn, + })); - Protocol { name, data } + Protocol { name, data } + } } fn register(&self, ctx: *mut libmpv2_sys::mpv_handle) -> Result<()> { diff --git a/crates/libmpv2/src/mpv/raw_error_warning.txt b/crates/libmpv2/src/mpv/raw_error_warning.txt new file mode 100644 index 0000000..277500a --- /dev/null +++ b/crates/libmpv2/src/mpv/raw_error_warning.txt @@ -0,0 +1,5 @@ +Raw mpv error: {} + +This error is directly returned from `mpv`. +This is probably caused by a bug in `yt`, please open an issue about +this and try to replicate it with the `-vvvv` verbosity setting. diff --git a/crates/libmpv2/src/mpv/raw_error_warning.txt.license b/crates/libmpv2/src/mpv/raw_error_warning.txt.license new file mode 100644 index 0000000..7813eb6 --- /dev/null +++ b/crates/libmpv2/src/mpv/raw_error_warning.txt.license @@ -0,0 +1,9 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +SPDX-License-Identifier: GPL-3.0-or-later + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/crates/libmpv2/src/mpv/render.rs b/crates/libmpv2/src/mpv/render.rs index c3f2dc9..02f70bb 100644 --- a/crates/libmpv2/src/mpv/render.rs +++ b/crates/libmpv2/src/mpv/render.rs @@ -8,9 +8,9 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -use crate::{mpv::mpv_err, Error, Result}; +use crate::{Error, Result, mpv::mpv_err}; use std::collections::HashMap; -use std::ffi::{c_void, CStr}; +use std::ffi::{CStr, c_void}; use std::os::raw::c_int; use std::ptr; @@ -125,26 +125,30 @@ impl<C> From<&RenderParam<C>> for u32 { } unsafe extern "C" fn gpa_wrapper<GLContext>(ctx: *mut c_void, name: *const i8) -> *mut c_void { - if ctx.is_null() { - panic!("ctx for get_proc_address wrapper is NULL"); - } + unsafe { + if ctx.is_null() { + panic!("ctx for get_proc_address wrapper is NULL"); + } - let params: *mut OpenGLInitParams<GLContext> = ctx as _; - let params = &*params; - (params.get_proc_address)( - ¶ms.ctx, - CStr::from_ptr(name) - .to_str() - .expect("Could not convert function name to str"), - ) + let params: *mut OpenGLInitParams<GLContext> = ctx as _; + let params = &*params; + (params.get_proc_address)( + ¶ms.ctx, + CStr::from_ptr(name) + .to_str() + .expect("Could not convert function name to str"), + ) + } } unsafe extern "C" fn ru_wrapper<F: Fn() + Send + 'static>(ctx: *mut c_void) { - if ctx.is_null() { - panic!("ctx for render_update wrapper is NULL"); - } + unsafe { + if ctx.is_null() { + panic!("ctx for render_update wrapper is NULL"); + } - (*(ctx as *mut F))(); + (*(ctx as *mut F))(); + } } impl<C> From<OpenGLInitParams<C>> for libmpv2_sys::mpv_opengl_init_params { @@ -197,14 +201,18 @@ impl<C> From<RenderParam<C>> for libmpv2_sys::mpv_render_param { } unsafe fn free_void_data<T>(ptr: *mut c_void) { - drop(Box::<T>::from_raw(ptr as *mut T)); + unsafe { + drop(Box::<T>::from_raw(ptr as *mut T)); + } } unsafe fn free_init_params<C>(ptr: *mut c_void) { - let params = Box::from_raw(ptr as *mut libmpv2_sys::mpv_opengl_init_params); - drop(Box::from_raw( - params.get_proc_address_ctx as *mut OpenGLInitParams<C>, - )); + unsafe { + let params = Box::from_raw(ptr as *mut libmpv2_sys::mpv_opengl_init_params); + drop(Box::from_raw( + params.get_proc_address_ctx as *mut OpenGLInitParams<C>, + )); + } } impl RenderContext { diff --git a/crates/libmpv2/update.sh b/crates/libmpv2/update.sh index ecd5aa8..591684a 100755 --- a/crates/libmpv2/update.sh +++ b/crates/libmpv2/update.sh @@ -10,8 +10,4 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -cd "$(dirname "$0")" || exit 1 -[ "$1" = "upgrade" ] && cargo upgrade --incompatible -cargo update - ./libmpv2-sys/update.sh "$@" diff --git a/crates/bytes/.gitignore b/crates/termsize/.gitignore index 8876ea6..5bc2870 100644 --- a/crates/bytes/.gitignore +++ b/crates/termsize/.gitignore @@ -1,11 +1,12 @@ # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 softprops <d.tangren@gmail.com> +# SPDX-License-Identifier: MIT # # This file is part of Yt. # # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -/target +target +Cargo.lock diff --git a/crates/termsize/Cargo.toml b/crates/termsize/Cargo.toml new file mode 100644 index 0000000..10ab7ed --- /dev/null +++ b/crates/termsize/Cargo.toml @@ -0,0 +1,36 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2025 softprops <d.tangren@gmail.com> +# SPDX-License-Identifier: MIT +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +[package] +name = "termsize" +authors = [ + "softprops <d.tangren@gmail.com>", + "Benedikt Peetz <benedikt.peetz@b-peetz.de>", +] +description = "Retrieves terminal size" +repository = "https://github.com/softprops/termsize" +homepage = "https://github.com/softprops/termsize" +documentation = "http://softprops.github.io/termsize" +keywords = ["tty", "terminal", "term", "size", "dimensions"] +license = "MIT" +readme = "README.md" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +publish = false + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +winapi = { version = "0.3", features = ["handleapi", "fileapi", "wincon"] } + +[lints] +workspace = true diff --git a/crates/termsize/LICENSE b/crates/termsize/LICENSE new file mode 100644 index 0000000..78c7d8a --- /dev/null +++ b/crates/termsize/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2015-2024 Doug Tangren + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/crates/termsize/LICENSE.license b/crates/termsize/LICENSE.license new file mode 100644 index 0000000..3562ab9 --- /dev/null +++ b/crates/termsize/LICENSE.license @@ -0,0 +1,9 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2025 softprops <d.tangren@gmail.com> +SPDX-License-Identifier: MIT + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/crates/termsize/README.md b/crates/termsize/README.md new file mode 100644 index 0000000..305669b --- /dev/null +++ b/crates/termsize/README.md @@ -0,0 +1,51 @@ +<!-- +yt - A fully featured command line YouTube client + +Copyright (C) 2025 softprops <d.tangren@gmail.com> +SPDX-License-Identifier: MIT + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. +--> + +# termsize + +[](https://github.com/softprops/termsize/actions/workflows/ci.yml) +[](https://crates.io/crates/termsize) + +> because terminal size matters + +Termsize is a rust crate providing a multi-platform interface for resolving your +terminal's current size in rows and columns. On most unix systems, this is +similar invoking the [stty(1)](http://man7.org/linux/man-pages/man1/stty.1.html) +program, requesting the terminal size. + +## [Documentation](https://softprops.github.com/termsize) + +## install + +run `cargo add termsize` in your terminal or add the following to your +`Cargo.toml` file + +```toml +[dependencies] +termsize = "0.1" +``` + +## usage + +Termize provides one function, `get`, which returns a `termsize::Size` struct +exposing two fields: `rows` and `cols` representing the number of rows and +columns a a terminal's stdout supports. + +```rust +pub fn main() { + termsize::get().map(|{ rows, cols }| { + println!("rows {} cols {}", size.rows, size.cols) + }); +} +``` + +Doug Tangren (softprops) 2015-2024 diff --git a/crates/termsize/src/lib.rs b/crates/termsize/src/lib.rs new file mode 100644 index 0000000..69e7b78 --- /dev/null +++ b/crates/termsize/src/lib.rs @@ -0,0 +1,52 @@ +#![deny(missing_docs)] +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 softprops <d.tangren@gmail.com> +// SPDX-License-Identifier: MIT +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! Termsize is a tiny crate that provides a simple +//! interface for retrieving the current +//! [terminal interface](http://www.manpagez.com/man/4/tty/) size +//! +//! ```rust +//! extern crate termsize; +//! +//! termsize::get().map(|size| println!("rows {} cols {}", size.rows, size.cols)); +//! ``` + +/// Container for number of rows and columns +#[derive(Debug, Clone, Copy)] +pub struct Size { + /// number of rows + pub rows: u16, + /// number of columns + pub cols: u16, +} + +#[cfg(unix)] +#[path = "nix.rs"] +mod imp; + +#[cfg(windows)] +#[path = "win.rs"] +mod imp; + +#[cfg(not(any(unix, windows)))] +#[path = "other.rs"] +mod imp; + +pub use imp::get; + +#[cfg(test)] +mod tests { + use super::get; + #[test] + fn test_get() { + assert!(get().is_some()); + } +} diff --git a/crates/termsize/src/nix.rs b/crates/termsize/src/nix.rs new file mode 100644 index 0000000..d672f54 --- /dev/null +++ b/crates/termsize/src/nix.rs @@ -0,0 +1,100 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 softprops <d.tangren@gmail.com> +// SPDX-License-Identifier: MIT +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::io::IsTerminal; + +use self::super::Size; +use libc::{STDOUT_FILENO, TIOCGWINSZ, c_ushort, ioctl}; + +/// A representation of the size of the current terminal +#[repr(C)] +#[derive(Debug)] +struct UnixSize { + /// number of rows + pub rows: c_ushort, + /// number of columns + pub cols: c_ushort, + x: c_ushort, + y: c_ushort, +} + +/// Gets the current terminal size +#[must_use] +pub fn get() -> Option<Size> { + // http://rosettacode.org/wiki/Terminal_control/Dimensions#Library:_BSD_libc + if !std::io::stdout().is_terminal() { + return None; + } + let mut us = UnixSize { + rows: 0, + cols: 0, + x: 0, + y: 0, + }; + let r = unsafe { ioctl(STDOUT_FILENO, TIOCGWINSZ, &mut us) }; + if r == 0 { + Some(Size { + rows: us.rows, + cols: us.cols, + }) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::{super::Size, get}; + use std::process::{Command, Output, Stdio}; + + #[cfg(target_os = "macos")] + fn stty_size() -> Output { + Command::new("stty") + .arg("-f") + .arg("/dev/stderr") + .arg("size") + .stderr(Stdio::inherit()) + .output() + .expect("expected stty output") + } + + #[cfg(not(target_os = "macos"))] + fn stty_size() -> Output { + Command::new("stty") + .arg("-F") + .arg("/dev/stderr") + .arg("size") + .stderr(Stdio::inherit()) + .output() + .expect("expected stty output") + } + + #[test] + fn test_shell() { + let output = stty_size(); + assert!(output.status.success()); + let stdout = String::from_utf8(output.stdout).expect("expected utf8"); + let mut data = stdout.split_whitespace(); + let rs = data + .next() + .expect("expected row") + .parse::<u16>() + .expect("expected u16 col"); + let cs = data + .next() + .expect("expected col") + .parse::<u16>() + .expect("expected u16 col"); + if let Some(Size { rows, cols }) = get() { + assert_eq!(rows, rs); + assert_eq!(cols, cs); + } + } +} diff --git a/crates/termsize/src/other.rs b/crates/termsize/src/other.rs new file mode 100644 index 0000000..8a02f22 --- /dev/null +++ b/crates/termsize/src/other.rs @@ -0,0 +1,14 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 softprops <d.tangren@gmail.com> +// SPDX-License-Identifier: MIT +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +/// Gets the current terminal size +pub fn get() -> Option<super::Size> { + None +} diff --git a/crates/termsize/src/win.rs b/crates/termsize/src/win.rs new file mode 100644 index 0000000..72d8433 --- /dev/null +++ b/crates/termsize/src/win.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 softprops <d.tangren@gmail.com> +// SPDX-License-Identifier: MIT +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::ptr; + +use winapi::um::{ + fileapi::{CreateFileA, OPEN_EXISTING}, + handleapi::INVALID_HANDLE_VALUE, + wincon::{CONSOLE_SCREEN_BUFFER_INFO, GetConsoleScreenBufferInfo}, + winnt::{FILE_SHARE_WRITE, GENERIC_READ, GENERIC_WRITE}, +}; + +use self::super::Size; + +/// Gets the current terminal size +pub fn get() -> Option<Size> { + // http://rosettacode.org/wiki/Terminal_control/Dimensions#Windows + let handle = unsafe { + CreateFileA( + b"CONOUT$\0".as_ptr() as *const i8, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_WRITE, + ptr::null_mut(), + OPEN_EXISTING, + 0, + ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return None; + } + let info = unsafe { + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms683171(v=vs.85).aspx + let mut info = ::std::mem::MaybeUninit::<CONSOLE_SCREEN_BUFFER_INFO>::uninit(); + if GetConsoleScreenBufferInfo(handle, info.as_mut_ptr()) == 0 { + None + } else { + Some(info.assume_init()) + } + }; + info.map(|inf| Size { + rows: (inf.srWindow.Bottom - inf.srWindow.Top + 1) as u16, + cols: (inf.srWindow.Right - inf.srWindow.Left + 1) as u16, + }) +} diff --git a/crates/yt/Cargo.toml b/crates/yt/Cargo.toml new file mode 100644 index 0000000..0b6c581 --- /dev/null +++ b/crates/yt/Cargo.toml @@ -0,0 +1,67 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +[package] +name = "yt" +description = "A fully featured command line YouTube client" +keywords = [] +categories = [] +default-run = "yt" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +anyhow = "1.0.98" +blake3 = { version = "1.8.2", features = ["serde"] } +chrono = { version = "0.4.41", features = ["now"] } +chrono-humanize = "0.2.3" +clap = { version = "4.5.41", features = ["derive"] } +clap_complete = { version = "4.5.55", features = ["unstable-dynamic"] } +colors.workspace = true +futures = "0.3.31" +libmpv2.workspace = true +log.workspace = true +notify = { version = "8.1.0", default-features = false } +regex = "1.11.1" +serde.workspace = true +serde_json.workspace = true +shlex = "1.3.0" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "sqlite"] } +stderrlog = "0.6.0" +tempfile = "3.20.0" +termsize.workspace = true +tokio-util = { version = "0.7.15", features = ["rt"] } +tokio.workspace = true +toml = "0.9.2" +url.workspace = true +uu_fmt.workspace = true +xdg = "3.0.0" +yt_dlp.workspace = true +reqwest = "0.12.22" + +[[bin]] +name = "yt" +doc = false +path = "src/main.rs" + +[lints] +workspace = true + +[dev-dependencies] +pretty_assertions = "1.4.1" + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/yt/src/ansi_escape_codes.rs b/crates/yt/src/ansi_escape_codes.rs new file mode 100644 index 0000000..28a8370 --- /dev/null +++ b/crates/yt/src/ansi_escape_codes.rs @@ -0,0 +1,29 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// see: https://en.wikipedia.org/wiki/ANSI_escape_code#Control_Sequence_Introducer_commands +const CSI: &str = "\x1b["; +pub(crate) fn erase_from_cursor_to_bottom() { + print!("{CSI}0J"); +} +pub(crate) fn cursor_up(number: usize) { + // HACK(@bpeetz): The default is `1` and running this command with a + // number of `0` results in it using the default (i.e., `1`) <2025-03-25> + if number != 0 { + print!("{CSI}{number}A"); + } +} + +pub(crate) fn clear_whole_line() { + eprint!("{CSI}2K"); +} +pub(crate) fn move_to_col(x: usize) { + eprint!("{CSI}{x}G"); +} diff --git a/crates/yt/src/app.rs b/crates/yt/src/app.rs new file mode 100644 index 0000000..3ea12a4 --- /dev/null +++ b/crates/yt/src/app.rs @@ -0,0 +1,50 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::{Context, Result}; +use log::warn; +use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; + +use crate::{config::Config, storage::migrate::migrate_db}; + +#[derive(Debug)] +pub(crate) struct App { + pub(crate) database: SqlitePool, + pub(crate) config: Config, +} + +impl App { + pub(crate) async fn new(config: Config, should_migrate_db: bool) -> Result<Self> { + let options = SqliteConnectOptions::new() + .filename(&config.paths.database_path) + .optimize_on_close(true, None) + .create_if_missing(true); + + let pool = SqlitePool::connect_with(options) + .await + .context("Failed to connect to database!")?; + + let app = App { + database: pool, + config, + }; + + if should_migrate_db { + migrate_db(&app) + .await + .context("Failed to migrate db to new version")?; + } else { + warn!("Skipping database migration."); + } + + Ok(app) + } +} diff --git a/crates/yt/src/cli.rs b/crates/yt/src/cli.rs new file mode 100644 index 0000000..9a24403 --- /dev/null +++ b/crates/yt/src/cli.rs @@ -0,0 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::PathBuf; + +use clap::{ArgAction, Parser}; + +use crate::commands::Command; + +#[derive(Parser, Debug)] +#[clap(author, about, long_about = None)] +#[allow(clippy::module_name_repetitions)] +/// An command line interface to select, download and watch videos +pub(crate) struct CliArgs { + #[command(subcommand)] + /// The subcommand to execute [default: select] + pub(crate) command: Option<Command>, + + /// Show the version and exit + #[arg(long, short = 'V', action= ArgAction::SetTrue)] + pub(crate) version: bool, + + /// Do not perform database migration before starting. + /// Setting this could cause runtime database access errors. + #[arg(long, short, action=ArgAction::SetTrue, default_value_t = false)] + pub(crate) no_migrate_db: bool, + + /// Display colors [defaults to true, if the config file has no value] + #[arg(long, short = 'C')] + pub(crate) color: Option<bool>, + + /// Set the path to the videos.db. This overrides the default and the config file. + #[arg(long, short)] + pub(crate) db_path: Option<PathBuf>, + + /// Set the path to the config.toml. + /// This overrides the default. + #[arg(long, short)] + pub(crate) config_path: Option<PathBuf>, + + /// Increase message verbosity + #[arg(long="verbose", short = 'v', action = ArgAction::Count)] + pub(crate) verbosity: u8, + + /// Silence all output + #[arg(long, short = 'q')] + pub(crate) quiet: bool, +} + +#[cfg(test)] +mod test { + use clap::CommandFactory; + + use super::CliArgs; + #[test] + fn verify_cli() { + CliArgs::command().debug_assert(); + } +} diff --git a/crates/yt/src/commands/cache/implm.rs b/crates/yt/src/commands/cache/implm.rs new file mode 100644 index 0000000..fd0fbce --- /dev/null +++ b/crates/yt/src/commands/cache/implm.rs @@ -0,0 +1,40 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::CacheCommand, + storage::db::{ + insert::Operations, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +impl CacheCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + CacheCommand::Clear {} => { + let mut videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("Cache clear"); + + for vid in &mut videos { + vid.remove_download_path(&mut ops); + } + + ops.commit(app).await?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/cache/mod.rs b/crates/yt/src/commands/cache/mod.rs new file mode 100644 index 0000000..4ed4b40 --- /dev/null +++ b/crates/yt/src/commands/cache/mod.rs @@ -0,0 +1,19 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Subcommand; + +mod implm; + +#[derive(Debug, Subcommand)] +pub(super) enum CacheCommand { + /// Remove all downloaded video files. + Clear {}, +} diff --git a/crates/yt/src/commands/config/implm.rs b/crates/yt/src/commands/config/implm.rs new file mode 100644 index 0000000..00c28a9 --- /dev/null +++ b/crates/yt/src/commands/config/implm.rs @@ -0,0 +1,23 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{app::App, commands::config::ConfigCommand}; + +use anyhow::Result; + +impl ConfigCommand { + pub(in crate::commands) fn implm(self, app: &App) -> Result<()> { + let config_str = toml::to_string(&app.config)?; + + print!("{config_str}"); + + Ok(()) + } +} diff --git a/crates/yt/src/commands/config/mod.rs b/crates/yt/src/commands/config/mod.rs new file mode 100644 index 0000000..503b4f7 --- /dev/null +++ b/crates/yt/src/commands/config/mod.rs @@ -0,0 +1,16 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct ConfigCommand {} diff --git a/crates/yt/src/commands/database/implm.rs b/crates/yt/src/commands/database/implm.rs new file mode 100644 index 0000000..07d346b --- /dev/null +++ b/crates/yt/src/commands/database/implm.rs @@ -0,0 +1,45 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::DatabaseCommand, + storage::db::{ + insert::{Committable, subscription, video}, + txn_log::TxnLog, + }, +}; + +use anyhow::Result; + +impl DatabaseCommand { + pub(in crate::commands) async fn implm(&self, app: &App) -> Result<()> { + match self { + DatabaseCommand::Log { kind } => match kind { + super::OperationType::Video => { + let log = TxnLog::<video::Operation>::get(app).await?; + display_log(&log); + } + super::OperationType::Subscription => { + let log = TxnLog::<subscription::Operation>::get(app).await?; + display_log(&log); + } + }, + } + + Ok(()) + } +} + +fn display_log<O: Committable>(log: &TxnLog<O>) { + for (time, value) in log.inner() { + println!("At {time}: {value:?}"); + } +} diff --git a/crates/yt/src/commands/database/mod.rs b/crates/yt/src/commands/database/mod.rs new file mode 100644 index 0000000..06e3169 --- /dev/null +++ b/crates/yt/src/commands/database/mod.rs @@ -0,0 +1,41 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{self, Display}; + +use clap::{Subcommand, ValueEnum}; + +mod implm; + +#[derive(Subcommand, Debug)] +pub(super) enum DatabaseCommand { + /// Show the history of operations, in they groups they were committed in. + Log { + /// What kind of operation to show. + #[arg(short, long, default_value_t)] + kind: OperationType, + }, +} + +#[derive(Debug, Clone, Copy, ValueEnum, Default)] +pub(super) enum OperationType { + #[default] + Video, + Subscription, +} + +impl Display for OperationType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OperationType::Video => f.write_str("video"), + OperationType::Subscription => f.write_str("subscription"), + } + } +} diff --git a/crates/yt/src/commands/download/implm/download/download_options.rs b/crates/yt/src/commands/download/implm/download/download_options.rs new file mode 100644 index 0000000..15fed7e --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/download_options.rs @@ -0,0 +1,121 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Context; +use serde_json::{Value, json}; +use yt_dlp::{YoutubeDL, options::YoutubeDLOptions}; + +use crate::app::App; + +use super::progress_hook::wrapped_progress_hook; + +pub(crate) fn download_opts( + app: &App, + subtitle_langs: Option<&String>, +) -> anyhow::Result<YoutubeDL> { + YoutubeDLOptions::new() + .with_progress_hook(wrapped_progress_hook) + .set("extract_flat", "in_playlist") + .set( + "extractor_args", + json! { + { + "youtube": { + "comment_sort": [ "top" ], + "max_comments": [ "150", "all", "100" ] + } + } + }, + ) + //.set("cookiesfrombrowser", json! {("firefox", "me.google", None::<String>, "youtube_dlp")}) + .set("prefer_free_formats", true) + .set("ffmpeg_location", env!("FFMPEG_LOCATION")) + .set("format", "bestvideo[height<=?1080]+bestaudio/best") + .set("fragment_retries", 10) + .set("getcomments", true) + .set("ignoreerrors", false) + .set("retries", 10) + .set("writeinfojson", true) + // NOTE: This results in a constant warning message. <2025-01-04> + //.set("writeannotations", true) + .set("writesubtitles", true) + .set("writeautomaticsub", true) + .set( + "outtmpl", + json! { + { + "default": app.config.paths.download_dir.join("%(channel)s/%(title)s.%(ext)s"), + "chapter": "%(title)s - %(section_number)03d %(section_title)s [%(id)s].%(ext)s" + } + }, + ) + .set("compat_opts", json! {{}}) + .set("forceprint", json! {{}}) + .set("print_to_file", json! {{}}) + .set("windowsfilenames", false) + .set("restrictfilenames", false) + .set("trim_file_names", false) + .set( + "postprocessors", + json! { + [ + { + "api": "https://sponsor.ajay.app", + "categories": [ + "interaction", + "intro", + "music_offtopic", + "sponsor", + "outro", + "poi_highlight", + "preview", + "selfpromo", + "filler", + "chapter" + ], + "key": "SponsorBlock", + "when": "after_filter" + }, + { + "force_keyframes": false, + "key": "ModifyChapters", + "remove_chapters_patterns": [], + "remove_ranges": [], + "remove_sponsor_segments": [ "sponsor" ], + "sponsorblock_chapter_title": "[SponsorBlock]: %(category_names)l" + }, + { + "add_chapters": true, + "add_infojson": null, + "add_metadata": false, + "key": "FFmpegMetadata" + }, + { + "key": "FFmpegConcat", + "only_multi_video": true, + "when": "playlist" + } + ] + }, + ) + .set( + "subtitleslangs", + Value::Array( + subtitle_langs + .map_or("", String::as_str) + .split(',') + .map(|val| Value::String(val.to_owned())) + .collect::<Vec<_>>(), + ), + ) + .build() + .context("Failed to instanciate download yt_dlp") +} diff --git a/crates/yt/src/commands/download/implm/download/mod.rs b/crates/yt/src/commands/download/implm/download/mod.rs new file mode 100644 index 0000000..876d6e6 --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/mod.rs @@ -0,0 +1,290 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; + +use crate::{ + app::App, + commands::download::implm::download::download_options::download_opts, + shared::bytes::Bytes, + storage::{ + db::{extractor_hash::ExtractorHash, insert::Operations, video::Video}, + notify::{wait_for_cache_reduction, wait_for_db_write}, + }, + yt_dlp::get_current_cache_allocation, +}; + +use anyhow::{Context, Result, bail}; +use log::{debug, error, info, warn}; +use tokio::{select, task::JoinHandle, time}; +use yt_dlp::YoutubeDL; + +#[allow(clippy::module_name_repetitions)] +pub(crate) mod download_options; +pub(crate) mod progress_hook; + +#[derive(Debug)] +#[allow(clippy::module_name_repetitions)] +pub(crate) struct CurrentDownload { + task_handle: JoinHandle<Result<(PathBuf, Video)>>, + yt_dlp: Arc<YoutubeDL>, + extractor_hash: ExtractorHash, +} + +impl CurrentDownload { + fn new_from_video(app: &App, video: Video) -> Result<Self> { + let extractor_hash = video.extractor_hash; + + debug!("Download started: {}", &video.title); + let yt_dlp = Arc::new(download_opts(app, video.subtitle_langs.as_ref())?); + + let local_yt_dlp = Arc::clone(&yt_dlp); + + let task_handle = tokio::task::spawn_blocking(move || { + let mut result = local_yt_dlp + .download(&[video.url.clone()]) + .with_context(|| format!("Failed to download video: '{}'", video.title))?; + + assert_eq!(result.len(), 1); + Ok((result.remove(0), video)) + }); + + Ok(Self { + task_handle, + yt_dlp, + extractor_hash, + }) + } + + fn abort(self) -> Result<()> { + debug!("Cancelling download."); + self.yt_dlp.close()?; + + Ok(()) + } + + fn is_finished(&self) -> bool { + self.task_handle.is_finished() + } + + async fn finalize(self, app: &App) -> Result<()> { + let (result, mut video) = self.task_handle.await??; + + let mut ops = Operations::new("Downloader: Set download path"); + video.set_download_path(&result, &mut ops); + ops.commit(app) + .await + .with_context(|| format!("Failed to committ download of video: '{}'", video.title))?; + + info!( + "Video '{}' was downlaoded to path: {}", + video.title, + result.display() + ); + + Ok(()) + } +} + +enum CacheSizeCheck { + /// The video can be downloaded + Fits, + + /// The video and the current cache size together would exceed the size + TooLarge, + + /// The video would not even fit into the empty cache + ExceedsMaxCacheSize, +} + +#[derive(Debug)] +pub(crate) struct Downloader { + current_download: Option<CurrentDownload>, + video_size_cache: HashMap<ExtractorHash, u64>, + printed_warning: bool, + cached_cache_allocation: Option<Bytes>, +} + +impl Default for Downloader { + fn default() -> Self { + Self::new() + } +} + +impl Downloader { + #[must_use] + pub(crate) fn new() -> Self { + Self { + current_download: None, + video_size_cache: HashMap::new(), + printed_warning: false, + cached_cache_allocation: None, + } + } + + /// Check if enough cache is available. + /// + /// Will wait for the next cache deletion if not. + async fn is_enough_cache_available( + &mut self, + app: &App, + max_cache_size: u64, + next_video: &Video, + ) -> Result<CacheSizeCheck> { + if let Some(cdownload) = &self.current_download { + if cdownload.extractor_hash == next_video.extractor_hash { + // If the video is already being downloaded it will always fit. Otherwise the + // download would not have been started. + return Ok(CacheSizeCheck::Fits); + } + } + let cache_allocation = get_current_cache_allocation(app).await?; + let video_size = self.get_approx_video_size(next_video)?; + + if video_size >= max_cache_size { + error!( + "The video '{}' ({}) exceeds the maximum cache size ({})! \ + Please set a bigger maximum (`--max-cache-size`) or skip it.", + next_video.title, + Bytes::new(video_size), + Bytes::new(max_cache_size) + ); + + return Ok(CacheSizeCheck::ExceedsMaxCacheSize); + } + + if cache_allocation.as_u64() + video_size >= max_cache_size { + if !self.printed_warning { + warn!( + "Can't download video: '{}' ({}) as it's too large for the cache ({} of {} allocated). \ + Waiting for cache size reduction..", + next_video.title, + Bytes::new(video_size), + &cache_allocation, + Bytes::new(max_cache_size) + ); + self.printed_warning = true; + + // Update this value immediately. + // This avoids printing the "Current cache size has changed .." warning below. + self.cached_cache_allocation = Some(cache_allocation); + } + + if let Some(cca) = self.cached_cache_allocation { + if cca != cache_allocation { + // Only print the warning if the display string has actually changed. + // Otherwise, we might confuse the user + if cca.to_string() != cache_allocation.to_string() { + warn!("Current cache size has changed, it's now: '{cache_allocation}'"); + } + debug!( + "Cache size has changed: {} -> {}", + cca.as_u64(), + cache_allocation.as_u64() + ); + self.cached_cache_allocation = Some(cache_allocation); + } + } else { + unreachable!( + "The `printed_warning` should be false in this case, \ + and thus should have already set the `cached_cache_allocation`." + ); + } + + // Wait and hope, that a large video is deleted from the cache. + wait_for_cache_reduction(app).await?; + Ok(CacheSizeCheck::TooLarge) + } else { + self.printed_warning = false; + Ok(CacheSizeCheck::Fits) + } + } + + /// The entry point to the Downloader. + /// This Downloader will periodically check if the database has changed, and then also + /// change which videos it downloads. + /// This will run, until the database doesn't contain any watchable videos + pub(crate) async fn consume(&mut self, app: Arc<App>, max_cache_size: u64) -> Result<()> { + while let Some(next_video) = Video::next_to_download(&app).await? { + match self + .is_enough_cache_available(&app, max_cache_size, &next_video) + .await? + { + CacheSizeCheck::Fits => (), + CacheSizeCheck::TooLarge => continue, + CacheSizeCheck::ExceedsMaxCacheSize => bail!("Giving up."), + } + + if self.current_download.is_some() { + let current_download = self.current_download.take().expect("It is `Some`."); + + if current_download.is_finished() { + // The download is done, finalize it and leave it removed. + current_download.finalize(&app).await?; + continue; + } + + if next_video.extractor_hash == current_download.extractor_hash { + // We still want to download the same video. + // reset the taken value + self.current_download = Some(current_download); + } else { + info!( + "Noticed, that the next video is not the video being downloaded, replacing it ('{}' vs. '{}')!", + next_video.extractor_hash.as_short_hash(&app).await?, + current_download.extractor_hash.as_short_hash(&app).await? + ); + + // Replace the currently downloading video + current_download + .abort() + .context("Failed to abort last download")?; + + let new_current_download = CurrentDownload::new_from_video(&app, next_video)?; + + self.current_download = Some(new_current_download); + } + } else { + info!( + "No video is being downloaded right now, setting it to '{}'", + next_video.title + ); + let new_current_download = CurrentDownload::new_from_video(&app, next_video)?; + self.current_download = Some(new_current_download); + } + + // We have to continuously check, if the current download is done. + // As such we simply wait or recheck on the next write to the db. + select! { + () = time::sleep(Duration::from_secs(1)) => (), + Ok(()) = wait_for_db_write(&app) => (), + } + } + + info!("Finished downloading!"); + Ok(()) + } + + fn get_approx_video_size(&mut self, video: &Video) -> Result<u64> { + if let Some(value) = self.video_size_cache.get(&video.extractor_hash) { + Ok(*value) + } else { + let size = video.get_approx_size()?; + + assert_eq!( + self.video_size_cache.insert(video.extractor_hash, size), + None + ); + + Ok(size) + } + } +} diff --git a/crates/yt/src/commands/download/implm/download/progress_hook.rs b/crates/yt/src/commands/download/implm/download/progress_hook.rs new file mode 100644 index 0000000..19fe122 --- /dev/null +++ b/crates/yt/src/commands/download/implm/download/progress_hook.rs @@ -0,0 +1,175 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + io::{Write, stderr}, + process, + sync::atomic::Ordering, +}; + +use colors::{Colorize, IntoCanvas}; +use log::{Level, log_enabled}; +use yt_dlp::{json_cast, json_get, wrap_progress_hook}; + +use crate::{ + ansi_escape_codes::{clear_whole_line, move_to_col}, + config::SHOULD_DISPLAY_COLOR, + select::duration::MaybeDuration, + shared::bytes::Bytes, +}; + +macro_rules! json_get_default { + ($value:expr, $name:literal, $convert:ident, $default:expr) => { + $value.get($name).map_or($default, |v| { + if v == &serde_json::Value::Null { + $default + } else { + json_cast!(@log_key $name, v, $convert) + } + }) + }; +} + +fn format_bytes(bytes: u64) -> String { + let bytes = Bytes::new(bytes); + bytes.to_string() +} + +fn format_speed(speed: f64) -> String { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let bytes = Bytes::new(speed.floor() as u64); + format!("{bytes}/s") +} + +/// # Panics +/// If expectations fail. +#[allow(clippy::needless_pass_by_value)] +pub(crate) fn progress_hook( + input: serde_json::Map<String, serde_json::Value>, +) -> Result<(), std::io::Error> { + // Only add the handler, if the log-level is higher than Debug (this avoids covering debug + // messages). + if log_enabled!(Level::Debug) { + return Ok(()); + } + + let info_dict = json_get!(input, "info_dict", as_object); + + let get_title = || -> String { + match json_get!(info_dict, "ext", as_str) { + "vtt" => { + format!( + "Subtitles ({})", + json_get_default!(info_dict, "name", as_str, "<No Subtitle Language>") + ) + } + "webm" | "mp4" | "mp3" | "m4a" => { + json_get_default!(info_dict, "title", as_str, "<No title>").to_owned() + } + other => panic!("The extension '{other}' is not yet implemented"), + } + }; + + match json_get!(input, "status", as_str) { + "downloading" => { + let elapsed = json_get_default!(input, "elapsed", as_f64, 0.0); + let eta = json_get_default!(input, "eta", as_f64, 0.0); + let speed = json_get_default!(input, "speed", as_f64, 0.0); + + let downloaded_bytes = json_get!(input, "downloaded_bytes", as_u64); + let (total_bytes, bytes_is_estimate): (u64, &'static str) = { + let total_bytes = json_get_default!(input, "total_bytes", as_u64, 0); + + if total_bytes == 0 { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let maybe_estimate = + json_get_default!(input, "total_bytes_estimate", as_f64, 0.0) as u64; + + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + if maybe_estimate == 0 { + // The download speed should be in bytes + // per second and the eta in seconds. + // Thus multiplying them gets us the raw bytes + // (which were estimated by `yt_dlp`, from their `info.json`) + let bytes_still_needed = (speed * eta).ceil() as u64; + + (downloaded_bytes + bytes_still_needed, "~") + } else { + (maybe_estimate, "~") + } + } else { + (total_bytes, "") + } + }; + + let percent: f64 = { + if total_bytes == 0 { + 100.0 + } else { + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + { + (downloaded_bytes as f64 / total_bytes as f64) * 100.0 + } + } + }; + + clear_whole_line(); + move_to_col(1); + + let should_use_color = SHOULD_DISPLAY_COLOR.load(Ordering::Relaxed); + + eprint!( + "{} [{}/{} at {}] -> [{} of {}{} {}] ", + get_title().bold().blue().render(should_use_color), + MaybeDuration::from_secs_f64(elapsed) + .bold() + .yellow() + .render(should_use_color), + MaybeDuration::from_secs_f64(eta) + .bold() + .yellow() + .render(should_use_color), + format_speed(speed).bold().green().render(should_use_color), + format_bytes(downloaded_bytes) + .bold() + .red() + .render(should_use_color), + bytes_is_estimate.bold().red().render(should_use_color), + format_bytes(total_bytes) + .bold() + .red() + .render(should_use_color), + format!("{percent:.02}%") + .bold() + .cyan() + .render(should_use_color), + ); + stderr().flush()?; + } + "finished" => { + eprintln!("-> Finished downloading."); + } + "error" => { + // TODO: This should probably return an Err. But I'm not so sure where the error would + // bubble up to (i.e., who would catch it) <2025-01-21> + eprintln!("-> Error while downloading: {}", get_title()); + process::exit(1); + } + other => unreachable!("'{other}' should not be a valid state!"), + } + + Ok(()) +} + +wrap_progress_hook!(progress_hook, wrapped_progress_hook); diff --git a/crates/yt/src/commands/download/implm/mod.rs b/crates/yt/src/commands/download/implm/mod.rs new file mode 100644 index 0000000..c74a909 --- /dev/null +++ b/crates/yt/src/commands/download/implm/mod.rs @@ -0,0 +1,55 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::Arc; + +use crate::{ + app::App, + commands::download::DownloadCommand, + shared::bytes::Bytes, + storage::db::{ + insert::{Operations, maintenance::clear_stale_downloaded_paths}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; +use log::info; + +mod download; + +impl DownloadCommand { + pub(in crate::commands) async fn implm(self, app: Arc<App>) -> Result<()> { + let DownloadCommand { + force, + max_cache_size, + } = self; + + let max_cache_size = max_cache_size.unwrap_or(app.config.download.max_cache_size.as_u64()); + info!("Max cache size: '{}'", Bytes::new(max_cache_size)); + + clear_stale_downloaded_paths(&app).await?; + if force { + let mut all = Video::in_states(&app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("Download: Clear old download paths due to `--force`"); + for a in &mut all { + a.remove_download_path(&mut ops); + } + ops.commit(&app).await?; + } + + download::Downloader::new() + .consume(app, max_cache_size) + .await?; + + Ok(()) + } +} diff --git a/crates/yt/src/commands/download/mod.rs b/crates/yt/src/commands/download/mod.rs new file mode 100644 index 0000000..15026ba --- /dev/null +++ b/crates/yt/src/commands/download/mod.rs @@ -0,0 +1,34 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Context; +use clap::Parser; + +use crate::shared::bytes::Bytes; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct DownloadCommand { + /// Forcefully re-download all cached videos (i.e. delete all already downloaded paths, then download). + #[arg(short, long)] + force: bool, + + /// The maximum size the download dir should have. + #[arg(short, long, value_parser = byte_parser)] + max_cache_size: Option<u64>, +} + +fn byte_parser(input: &str) -> Result<u64, anyhow::Error> { + Ok(input + .parse::<Bytes>() + .with_context(|| format!("Failed to parse '{input}' as bytes!"))? + .as_u64()) +} diff --git a/crates/yt/src/commands/implm.rs b/crates/yt/src/commands/implm.rs new file mode 100644 index 0000000..7c60c6a --- /dev/null +++ b/crates/yt/src/commands/implm.rs @@ -0,0 +1,38 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::Arc; + +use crate::commands::Command; + +use anyhow::Result; + +impl Command { + pub(crate) async fn implm(self, app: crate::app::App) -> Result<()> { + match self { + Command::Cache { cmd } => cmd.implm(&app).await?, + Command::Config { cmd } => cmd.implm(&app)?, + Command::Database { cmd } => cmd.implm(&app).await?, + Command::Download { cmd } => cmd.implm(Arc::new(app)).await?, + Command::Playlist { cmd } => cmd.implm(&app).await?, + Command::Select { cmd } => { + cmd.unwrap_or_default().implm(&app).await?; + } + Command::Show { cmd } => cmd.implm(&app).await?, + Command::Status { cmd } => cmd.implm(&app).await?, + Command::Subscriptions { cmd } => cmd.implm(&app).await?, + Command::Update { cmd } => cmd.implm(&app).await?, + Command::Videos { cmd } => cmd.implm(&app).await?, + Command::Watch { cmd } => cmd.implm(Arc::new(app)).await?, + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/mod.rs b/crates/yt/src/commands/mod.rs new file mode 100644 index 0000000..431acef --- /dev/null +++ b/crates/yt/src/commands/mod.rs @@ -0,0 +1,164 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ffi::OsStr, thread}; + +use clap::Subcommand; +use clap_complete::CompletionCandidate; +use tokio::runtime::Runtime; + +use crate::{ + app::App, + commands::{ + cache::CacheCommand, config::ConfigCommand, database::DatabaseCommand, + download::DownloadCommand, playlist::PlaylistCommand, select::SelectCommand, + show::ShowCommand, status::StatusCommand, subscriptions::SubscriptionCommand, + update::UpdateCommand, videos::VideosCommand, watch::WatchCommand, + }, + config::Config, + storage::db::subscription::Subscriptions, +}; + +pub(super) mod implm; + +mod cache; +mod config; +mod database; +mod download; +mod playlist; +mod select; +mod show; +mod status; +mod subscriptions; +mod update; +mod videos; +mod watch; + +#[derive(Subcommand, Debug)] +#[allow(private_interfaces)] // Only the main `implm` method should be accessible. +pub(super) enum Command { + /// Manipulate the download cache + Cache { + #[command(subcommand)] + cmd: CacheCommand, + }, + + /// Show, the configuration options in effect. + Config { + #[command(flatten)] + cmd: ConfigCommand, + }, + + /// Interact with the video database. + #[command(visible_alias = "db")] + Database { + #[command(subcommand)] + cmd: DatabaseCommand, + }, + + /// Download and cache URLs + Download { + #[command(flatten)] + cmd: DownloadCommand, + }, + + /// Visualize the current playlist + Playlist { + #[command(flatten)] + cmd: PlaylistCommand, + }, + + /// Change the state of videos in the database (the default) + Select { + #[command(subcommand)] + cmd: Option<SelectCommand>, + }, + + /// Show things about the currently playing video. + Show { + #[command(subcommand)] + cmd: ShowCommand, + }, + + /// Show, which videos have been selected to be watched (and their cache status) + Status { + #[command(flatten)] + cmd: StatusCommand, + }, + + /// Manipulate subscription + #[command(visible_alias = "subs")] + Subscriptions { + #[command(subcommand)] + cmd: SubscriptionCommand, + }, + + /// Update the video database + Update { + #[command(flatten)] + cmd: UpdateCommand, + }, + + /// Work with single videos + Videos { + #[command(subcommand)] + cmd: VideosCommand, + }, + + /// Watch the already cached (and selected) videos + Watch { + #[command(flatten)] + cmd: WatchCommand, + }, +} + +impl Default for Command { + fn default() -> Self { + Self::Select { + cmd: Some(SelectCommand::default()), + } + } +} + +fn complete_subscription(current: &OsStr) -> Vec<CompletionCandidate> { + let mut output = vec![]; + + let Some(current_prog) = current.to_str().map(ToOwned::to_owned) else { + return output; + }; + + let Ok(config) = Config::from_config_file(None, None, None) else { + return output; + }; + + let handle = thread::spawn(move || { + let Ok(rt) = Runtime::new() else { + return output; + }; + + let Ok(app) = rt.block_on(App::new(config, false)) else { + return output; + }; + + let Ok(all) = rt.block_on(Subscriptions::get(&app)) else { + return output; + }; + + for sub in all.0.into_keys() { + if sub.starts_with(¤t_prog) { + output.push(CompletionCandidate::new(sub)); + } + } + + output + }); + + handle.join().unwrap_or_default() +} diff --git a/crates/yt/src/commands/playlist/implm.rs b/crates/yt/src/commands/playlist/implm.rs new file mode 100644 index 0000000..603184b --- /dev/null +++ b/crates/yt/src/commands/playlist/implm.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{fmt::Write, path::Path}; + +use crate::{ + ansi_escape_codes, + app::App, + commands::playlist::PlaylistCommand, + storage::{ + db::{ + playlist::Playlist, + video::{Video, VideoStatus}, + }, + notify::wait_for_db_write, + }, + videos::RenderWithApp, +}; + +use anyhow::Result; +use futures::{TryStreamExt, stream::FuturesOrdered}; + +impl PlaylistCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let PlaylistCommand { watch } = self; + + let mut previous_output_length = 0; + loop { + let playlist = Playlist::create(app).await?.videos; + + let output = playlist + .into_iter() + .map(|video| async move { + let mut output = String::new(); + + let (_, is_focused) = cache_values(&video); + + if is_focused { + output.push_str("🔻 "); + } else { + output.push_str(" "); + } + + output.push_str(&video.title_fmt().to_string(app)); + + output.push_str(" ("); + output.push_str(&video.parent_subscription_name_fmt().to_string(app)); + output.push(')'); + + output.push_str(" ["); + output.push_str(&video.duration_fmt().to_string(app)); + + if is_focused { + output.push_str(" ("); + if let Some(percent) = video.watch_progress_percent_fmt() { + write!(output, "{}", percent.to_string(app))?; + } else { + write!(output, "{}", video.watch_progress_fmt().to_string(app))?; + } + + output.push(')'); + } + output.push(']'); + + output.push('\n'); + + Ok::<String, anyhow::Error>(output) + }) + .collect::<FuturesOrdered<_>>() + .try_collect::<String>() + .await?; + + // Delete the previous output + ansi_escape_codes::cursor_up(previous_output_length); + ansi_escape_codes::erase_from_cursor_to_bottom(); + + previous_output_length = output.chars().filter(|ch| *ch == '\n').count(); + + print!("{output}"); + + if !watch { + break; + } + + wait_for_db_write(app).await?; + } + + Ok(()) + } +} + +/// Extract the values of the [`VideoStatus::Cached`] value from a Video. +fn cache_values(video: &Video) -> (&Path, bool) { + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + { + (cache_path, *is_focused) + } else { + unreachable!("All of these videos should be cached"); + } +} diff --git a/crates/yt/src/commands/playlist/mod.rs b/crates/yt/src/commands/playlist/mod.rs new file mode 100644 index 0000000..8d3407d --- /dev/null +++ b/crates/yt/src/commands/playlist/mod.rs @@ -0,0 +1,20 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct PlaylistCommand { + /// Linger and display changes + #[arg(short, long)] + watch: bool, +} diff --git a/crates/yt/src/commands/select/implm/fs_generators/help.str b/crates/yt/src/commands/select/implm/fs_generators/help.str new file mode 100644 index 0000000..e3cc347 --- /dev/null +++ b/crates/yt/src/commands/select/implm/fs_generators/help.str @@ -0,0 +1,12 @@ +# Commands: +# w, watch [-p,-s,-l] Mark the video given by the hash to be watched +# wd, watched [-p,-s,-l] Mark the video given by the hash as already watched +# d, drop [-p,-s,-l] Mark the video given by the hash to be dropped +# u, url [-p,-s,-l] Open the video URL in Firefox's `timesinks.youtube` profile +# p, pick [-p,-s,-l] Reset the videos status to 'Pick' +# a, add URL Add a video, defined by the URL +# +# See `yt select <cmd_name> --help` for more help. +# +# These lines can be re-ordered; they are executed from top to bottom. +# vim: filetype=yts conceallevel=2 concealcursor=nc colorcolumn= nowrap diff --git a/crates/bytes/Cargo.lock.license b/crates/yt/src/commands/select/implm/fs_generators/help.str.license index d4d410f..a0e196c 100644 --- a/crates/bytes/Cargo.lock.license +++ b/crates/yt/src/commands/select/implm/fs_generators/help.str.license @@ -1,6 +1,7 @@ yt - A fully featured command line YouTube client Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> SPDX-License-Identifier: GPL-3.0-or-later This file is part of Yt. diff --git a/crates/yt/src/commands/select/implm/fs_generators/mod.rs b/crates/yt/src/commands/select/implm/fs_generators/mod.rs new file mode 100644 index 0000000..10da032 --- /dev/null +++ b/crates/yt/src/commands/select/implm/fs_generators/mod.rs @@ -0,0 +1,355 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::HashMap, + env, + fs::{self, File, OpenOptions}, + io::{BufRead, BufReader, BufWriter, Read, Write}, + iter, + os::fd::{AsFd, AsRawFd}, + path::Path, +}; + +use crate::{ + app::App, + cli::CliArgs, + commands::{ + Command, + select::{ + SelectCommand, SelectSplitSortKey, SelectSplitSortMode, + implm::standalone::{self, handle_select_cmd}, + }, + }, + storage::db::{ + extractor_hash::ExtractorHash, + insert::Operations, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use futures::{TryStreamExt, stream::FuturesOrdered}; +use log::info; +use shlex::Shlex; +use tokio::process; + +const HELP_STR: &str = include_str!("./help.str"); + +pub(crate) async fn select_split( + app: &App, + done: bool, + sort_key: SelectSplitSortKey, + sort_mode: SelectSplitSortMode, +) -> Result<()> { + let temp_dir = tempfile::Builder::new() + .prefix("yt_video_select-") + .rand_bytes(6) + .tempdir() + .context("Failed to get tempdir")?; + + let matching_videos = get_videos(app, done).await?; + + let mut no_author = vec![]; + let mut author_map = HashMap::new(); + for video in matching_videos { + if let Some(sub) = &video.parent_subscription_name { + if author_map.contains_key(sub) { + let vec: &mut Vec<_> = author_map + .get_mut(sub) + .expect("This key is set, we checked in the if above"); + + vec.push(video); + } else { + author_map.insert(sub.to_owned(), vec![video]); + } + } else { + no_author.push(video); + } + } + + let author_map = { + let mut temp_vec: Vec<_> = author_map.into_iter().collect(); + + match sort_key { + SelectSplitSortKey::Publisher => { + // PERFORMANCE: The clone here should not be neeed. <2025-06-15> + temp_vec.sort_by_key(|(name, _): &(String, Vec<Video>)| name.to_owned()); + } + SelectSplitSortKey::Videos => { + temp_vec.sort_by_key(|(_, videos): &(String, Vec<Video>)| videos.len()); + } + } + + match sort_mode { + SelectSplitSortMode::Asc => { + // Std's default mode is ascending. + } + SelectSplitSortMode::Desc => { + temp_vec.reverse(); + } + } + + temp_vec + }; + + for (index, (name, videos)) in author_map + .into_iter() + .chain(iter::once(( + "<No parent subscription>".to_owned(), + no_author, + ))) + .enumerate() + { + let mut file_path = temp_dir.path().join(format!("{index:02}_{name}")); + file_path.set_extension("yts"); + + let tmp_file = File::create(&file_path) + .with_context(|| format!("Falied to create file at: {}", file_path.display()))?; + + write_videos_to_file(app, &tmp_file, &videos) + .await + .with_context(|| format!("Falied to populate file at: {}", file_path.display()))?; + } + + open_editor_at(temp_dir.path()).await?; + + let mut paths = vec![]; + for maybe_entry in temp_dir + .path() + .read_dir() + .context("Failed to open temp dir for reading")? + { + let entry = maybe_entry.context("Failed to read entry in temp dir")?; + + if !entry.file_type()?.is_file() { + bail!("Found non-file entry: {}", entry.path().display()); + } + + paths.push(entry.path()); + } + + paths.sort(); + + let mut persistent_file = OpenOptions::new() + .read(false) + .write(true) + .create(true) + .truncate(true) + .open(&app.config.paths.last_selection_path) + .context("Failed to open persistent selection file")?; + + for path in paths { + let mut read_file = File::open(path)?; + + let mut buffer = vec![]; + read_file.read_to_end(&mut buffer)?; + persistent_file.write_all(&buffer)?; + } + + persistent_file.flush()?; + let persistent_file = OpenOptions::new() + .read(true) + .open(format!( + "/proc/self/fd/{}", + persistent_file.as_fd().as_raw_fd() + )) + .context("Failed to re-open persistent file")?; + + let processed = process_file(app, &persistent_file).await?; + + info!("Processed {processed} records."); + temp_dir.close().context("Failed to close the temp dir")?; + Ok(()) +} + +pub(crate) async fn select_file(app: &App, done: bool, use_last_selection: bool) -> Result<()> { + let temp_file = tempfile::Builder::new() + .prefix("yt_video_select-") + .suffix(".yts") + .rand_bytes(6) + .tempfile() + .context("Failed to get tempfile")?; + + if use_last_selection { + fs::copy(&app.config.paths.last_selection_path, &temp_file)?; + } else { + let matching_videos = get_videos(app, done).await?; + + write_videos_to_file(app, temp_file.as_file(), &matching_videos).await?; + } + + open_editor_at(temp_file.path()).await?; + + let read_file = OpenOptions::new().read(true).open(temp_file.path())?; + fs::copy(temp_file.path(), &app.config.paths.last_selection_path) + .context("Failed to persist selection file")?; + + let processed = process_file(app, &read_file).await?; + info!("Processed {processed} records."); + + Ok(()) +} + +async fn get_videos(app: &App, include_done: bool) -> Result<Vec<Video>> { + if include_done { + Video::in_states(app, VideoStatusMarker::ALL).await + } else { + Video::in_states( + app, + &[ + VideoStatusMarker::Pick, + // + VideoStatusMarker::Watch, + VideoStatusMarker::Cached, + ], + ) + .await + } +} + +async fn write_videos_to_file(app: &App, file: &File, videos: &[Video]) -> Result<()> { + // Warm-up the cache for the display rendering of the videos. + // Otherwise the futures would all try to warm it up at the same time. + if let Some(vid) = videos.first() { + drop(vid.to_line_display(app, None).await?); + } + + let mut edit_file = BufWriter::new(file); + + videos + .iter() + .map(|vid| vid.to_select_file_display(app)) + .collect::<FuturesOrdered<_>>() + .try_collect::<Vec<String>>() + .await? + .into_iter() + .try_for_each(|line| -> Result<()> { + edit_file + .write_all(line.as_bytes()) + .context("Failed to write to `edit_file`")?; + + Ok(()) + })?; + + edit_file.write_all(HELP_STR.as_bytes())?; + edit_file.flush().context("Failed to flush edit file")?; + + Ok(()) +} + +async fn process_file(app: &App, file: &File) -> Result<i64> { + let mut line_number = 0; + + let mut ops = Operations::new("Select: process file"); + + // Fetch all the hashes once, instead of every time we need to process a line. + let all_hashes = ExtractorHash::get_all(app).await?; + + let reader = BufReader::new(file); + for line in reader.lines() { + let line = line.context("Failed to read a line")?; + + if let Some(line) = process_line(&line)? { + line_number -= 1; + + // debug!( + // "Parsed command: `{}`", + // line.iter() + // .map(|val| format!("\"{}\"", val)) + // .collect::<Vec<String>>() + // .join(" ") + // ); + + let arg_line = ["yt", "select"] + .into_iter() + .chain(line.iter().map(String::as_str)); + + let args = CliArgs::parse_from(arg_line); + + let Command::Select { cmd } = args + .command + .expect("This will be some, as we constructed it above.") + else { + unreachable!("This is checked in the `filter_line` function") + }; + + match cmd.expect( + "This value should always be some \ + here, as it would otherwise thrown an error above.", + ) { + SelectCommand::File { .. } | SelectCommand::Split { .. } => { + bail!("You cannot use `select file` or `select split` recursively.") + } + SelectCommand::Add { urls, start, stop } => { + Box::pin(standalone::add::add(app, urls, start, stop)).await?; + } + other => { + let shared = other + .clone() + .into_shared() + .expect("The ones without shared should have been filtered out."); + + let hash = shared.hash.realize(app, Some(&all_hashes)).await?; + let mut video = hash + .get_with_app(app) + .await + .expect("The hash was already realized, it should therefore exist"); + + handle_select_cmd(app, other, &mut video, Some(line_number), &mut ops).await?; + } + } + } + } + + ops.commit(app).await?; + Ok(-line_number) +} + +async fn open_editor_at(path: &Path) -> Result<()> { + let editor = env::var("EDITOR").unwrap_or("nvim".to_owned()); + + let mut nvim = process::Command::new(&editor); + nvim.arg(path); + let status = nvim + .status() + .await + .with_context(|| format!("Falied to run editor: {editor}"))?; + + if status.success() { + Ok(()) + } else { + bail!("Editor ({editor}) exited with error status: {}", status) + } +} + +fn process_line(line: &str) -> Result<Option<Vec<String>>> { + // Filter out comments and empty lines + if line.starts_with('#') || line.trim().is_empty() { + Ok(None) + } else { + let split: Vec<_> = { + let mut shl = Shlex::new(line); + let res = shl.by_ref().collect(); + + if shl.had_error { + bail!("Failed to parse line '{line}'") + } + + assert_eq!(shl.line_no, 1, "A unexpected newline appeared"); + res + }; + + assert!(!split.is_empty()); + + Ok(Some(split)) + } +} diff --git a/crates/yt/src/commands/select/implm/mod.rs b/crates/yt/src/commands/select/implm/mod.rs new file mode 100644 index 0000000..f39c77f --- /dev/null +++ b/crates/yt/src/commands/select/implm/mod.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{app::App, commands::select::SelectCommand, storage::db::insert::Operations}; + +use anyhow::Result; + +mod fs_generators; +mod standalone; + +impl SelectCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + SelectCommand::File { + done, + use_last_selection, + } => Box::pin(fs_generators::select_file(app, done, use_last_selection)).await?, + SelectCommand::Split { + done, + sort_key, + sort_mode, + } => Box::pin(fs_generators::select_split(app, done, sort_key, sort_mode)).await?, + SelectCommand::Add { urls, start, stop } => { + Box::pin(standalone::add::add(app, urls, start, stop)).await?; + } + other => { + let shared = other + .clone() + .into_shared() + .expect("The ones without shared should have been filtered out."); + let hash = shared.hash.realize(app, None).await?; + let mut video = hash + .get_with_app(app) + .await + .expect("The hash was already realized, it should therefore exist"); + + let mut ops = Operations::new("Main: handle select cmd"); + standalone::handle_select_cmd(app, other, &mut video, None, &mut ops).await?; + ops.commit(app).await?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/select/implm/standalone/add.rs b/crates/yt/src/commands/select/implm/standalone/add.rs new file mode 100644 index 0000000..dd11cb4 --- /dev/null +++ b/crates/yt/src/commands/select/implm/standalone/add.rs @@ -0,0 +1,181 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{extractor_hash::ExtractorHash, insert::Operations, video::Video}, + yt_dlp::yt_dlp_opts_updating, +}; + +use anyhow::{Context, Result, bail}; +use log::{error, warn}; +use url::Url; +use yt_dlp::{YoutubeDL, info_json::InfoJson, json_cast, json_get, json_try_get}; + +#[allow(clippy::too_many_lines)] +pub(crate) async fn add( + app: &App, + urls: Vec<Url>, + start: Option<usize>, + stop: Option<usize>, +) -> Result<()> { + for url in urls { + async fn process_and_add(app: &App, entry: InfoJson, yt_dlp: &YoutubeDL) -> Result<()> { + let url = json_get!(entry, "url", as_str).parse()?; + + let entry = yt_dlp + .extract_info(&url, false, true) + .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; + + add_entry(app, entry).await?; + + Ok(()) + } + + async fn add_entry(app: &App, entry: InfoJson) -> Result<()> { + // We have to re-fetch all hashes every time, because a user could try to add the same + // URL twice (for whatever reason.) + let hashes = ExtractorHash::get_all(app) + .await + .context("Failed to fetch all video hashes")?; + + let extractor_hash = ExtractorHash::from_info_json(&entry); + if hashes.contains(&extractor_hash) { + error!( + "Video '{}'{} is already in the database. Skipped adding it", + extractor_hash + .as_short_hash(app) + .await + .with_context(|| format!( + "Failed to format hash of video '{}' as short hash", + json_try_get!(entry, "url", as_str).unwrap_or("<Unknown video Url>") + ))?, + json_try_get!(entry, "title", as_str) + .map_or(String::new(), |title| format!(" (\"{title}\")")) + ); + return Ok(()); + } + + let mut ops = Operations::new("SelectAdd: Video entry to video"); + let video = Video::from_info_json(&entry, None)?.add(&mut ops)?; + ops.commit(app).await?; + + println!("{}", &video.to_line_display(app, None).await?); + + Ok(()) + } + + let yt_dlp = yt_dlp_opts_updating(start.unwrap_or(0) + stop.unwrap_or(0))?; + + let entry = yt_dlp + .extract_info(&url, false, true) + .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; + + match json_try_get!(entry, "_type", as_str) { + Some("video") => { + add_entry(app, entry).await?; + if start.is_some() || stop.is_some() { + warn!( + "You added `start` and/or `stop` markers for a single *video*! These will be ignored." + ); + } + } + Some("playlist") => { + if let Some(entries) = json_try_get!(entry, "entries", as_array) { + let start = start.unwrap_or(0); + let stop = stop.unwrap_or(entries.len() - 1); + + let respected_entries = + take_vector(entries, start, stop).with_context(|| { + format!( + "Failed to take entries starting at: {start} and ending with {stop}" + ) + })?; + + if respected_entries.is_empty() { + warn!("No entries found, after applying your start/stop limits."); + } else { + // Pre-warm the cache + process_and_add( + app, + json_cast!(respected_entries[0], as_object).to_owned(), + &yt_dlp, + ) + .await?; + let respected_entries = &respected_entries[1..]; + + let futures: Vec<_> = respected_entries + .iter() + .map(|entry| { + process_and_add( + app, + json_cast!(entry, as_object).to_owned(), + &yt_dlp, + ) + }) + .collect(); + + for fut in futures { + fut.await?; + } + } + } else { + bail!("Your playlist does not seem to have any entries!") + } + } + other => bail!( + "Your URL should point to a video or a playlist, but points to a '{:#?}'", + other + ), + } + } + + Ok(()) +} + +fn take_vector<T>(vector: &[T], start: usize, stop: usize) -> Result<&[T]> { + let length = vector.len(); + + if stop >= length { + bail!( + "Your stop marker ({stop}) exceeds the possible entries ({length})! Remember that it is zero indexed." + ); + } + + Ok(&vector[start..=stop]) +} + +#[cfg(test)] +mod test { + use super::take_vector; + + #[test] + fn test_vector_take() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + let new_vec = take_vector(&vec, 2, 8).unwrap(); + + assert_eq!(new_vec, vec![2, 3, 4, 5, 6, 7, 8]); + } + + #[test] + fn test_vector_take_overflow() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + assert!(take_vector(&vec, 0, 12).is_err()); + } + + #[test] + fn test_vector_take_equal() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + assert!(take_vector(&vec, 0, 11).is_err()); + } +} diff --git a/crates/yt/src/commands/select/implm/standalone/mod.rs b/crates/yt/src/commands/select/implm/standalone/mod.rs new file mode 100644 index 0000000..9512e32 --- /dev/null +++ b/crates/yt/src/commands/select/implm/standalone/mod.rs @@ -0,0 +1,132 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::io::{Write, stderr}; + +use crate::{ + ansi_escape_codes, + app::App, + commands::select::{SelectCommand, SharedSelectionCommandArgs}, + storage::db::{ + insert::{Operations, video::Operation}, + video::{Priority, Video, VideoStatus}, + }, +}; + +use anyhow::{Context, Result, bail}; + +pub(super) mod add; + +pub(crate) async fn handle_select_cmd( + app: &App, + cmd: SelectCommand, + video: &mut Video, + line_number: Option<i64>, + ops: &mut Operations<Operation>, +) -> Result<()> { + let status = match cmd { + SelectCommand::Pick { shared } => Some((VideoStatus::Pick, shared)), + SelectCommand::Drop { shared } => Some((VideoStatus::Drop, shared)), + SelectCommand::Watched { shared } => Some((VideoStatus::Watched, shared)), + SelectCommand::Watch { shared } => { + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + { + Some(( + VideoStatus::Cached { + cache_path: cache_path.to_owned(), + is_focused: *is_focused, + }, + shared, + )) + } else { + Some((VideoStatus::Watch, shared)) + } + } + SelectCommand::Url { shared } => { + let Some(url) = shared.url else { + bail!("You need to provide a url to `select url ..`") + }; + + let mut firefox = std::process::Command::new(app.config.commands.url_opener.first()); + firefox.args(app.config.commands.url_opener.tail()); + firefox.arg(url.as_str()); + let _handle = firefox.spawn().context("Failed to run firefox")?; + None + } + SelectCommand::File { .. } | SelectCommand::Split { .. } | SelectCommand::Add { .. } => { + unreachable!("These should have been filtered out") + } + }; + + if let Some((status, shared)) = status { + handle_status_change( + app, + video, + shared, + line_number, + status, + line_number.is_none(), + ops, + ) + .await?; + } + + Ok(()) +} + +async fn handle_status_change( + app: &App, + video: &mut Video, + shared: SharedSelectionCommandArgs, + line_number: Option<i64>, + new_status: VideoStatus, + is_single: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + let priority = compute_priority(line_number, shared.priority); + + video.set_status(new_status, ops); + if let Some(priority) = priority { + video.set_priority(priority, ops); + } + + if let Some(subtitle_langs) = shared.subtitle_langs { + video.set_subtitle_langs(subtitle_langs, ops); + } + if let Some(playback_speed) = shared.playback_speed { + video.set_playback_speed(playback_speed, ops); + } + + if !is_single { + ansi_escape_codes::clear_whole_line(); + ansi_escape_codes::move_to_col(1); + } + + eprint!("{}", &video.to_line_display(app, None).await?); + + if is_single { + eprintln!(); + } else { + stderr().flush()?; + } + + Ok(()) +} + +fn compute_priority(line_number: Option<i64>, priority: Option<i64>) -> Option<Priority> { + if let Some(pri) = priority { + Some(Priority::from(pri)) + } else { + line_number.map(Priority::from) + } +} diff --git a/crates/yt/src/commands/select/mod.rs b/crates/yt/src/commands/select/mod.rs new file mode 100644 index 0000000..db69238 --- /dev/null +++ b/crates/yt/src/commands/select/mod.rs @@ -0,0 +1,230 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fmt::{self, Display, Formatter}, + str::FromStr, +}; + +use chrono::NaiveDate; +use clap::{Args, Subcommand, ValueEnum}; +use url::Url; + +use crate::{select::duration::MaybeDuration, storage::db::extractor_hash::LazyExtractorHash}; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +// NOTE: Keep this in sync with the [`constants::HELP_STR`] constant. <2024-08-20> +// NOTE: Also keep this in sync with the `tree-sitter-yts/grammar.js`. <2024-11-04> +#[allow(private_interfaces)] // Only the main `implm` method should be accessible. +pub(super) enum SelectCommand { + /// Open a `git rebase` like file to select the videos to watch (the default) + File { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Use the last selection file (useful if you've spend time on it and want to get it again) + #[arg(long, short, conflicts_with = "done")] + use_last_selection: bool, + }, + + /// Generate a directory, where each file contains only one subscription. + Split { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Which key to use for sorting. + #[arg(default_value_t)] + sort_key: SelectSplitSortKey, + + /// Which mode to use for sorting. + #[arg(default_value_t)] + sort_mode: SelectSplitSortMode, + }, + + /// Add a video to the database + /// + /// This optionally supports to add a playlist. + /// When a playlist is added, the `start` and `stop` arguments can be used to select which + /// playlist entries to include. + #[command(visible_alias = "a")] + Add { + urls: Vec<Url>, + + /// Start adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 's', long)] + start: Option<usize>, + + /// Stop adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 'e', long)] + stop: Option<usize>, + }, + + /// Mark the video given by the hash to be watched + #[command(visible_alias = "w")] + Watch { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash to be dropped + #[command(visible_alias = "d")] + Drop { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash as already watched + #[command(visible_alias = "wd")] + Watched { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Open the video URL in your specified command + #[command(visible_alias = "u")] + Url { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Reset the videos status to 'Pick' + #[command(visible_alias = "p")] + Pick { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, +} +impl Default for SelectCommand { + fn default() -> Self { + Self::File { + done: false, + use_last_selection: false, + } + } +} + +#[derive(Clone, Debug, Args)] +#[command(infer_subcommands = true)] +/// Mark the video given by the hash to be watched +struct SharedSelectionCommandArgs { + /// The ordering priority (higher means more at the top) + #[arg(short, long)] + priority: Option<i64>, + + /// The subtitles to download (e.g. 'en,de,sv') + #[arg(short = 'l', long)] + subtitle_langs: Option<String>, + + /// The speed to set mpv to + #[arg(short = 's', long)] + playback_speed: Option<f64>, + + /// The short extractor hash + hash: LazyExtractorHash, + + title: Option<String>, + + date: Option<OptionalNaiveDate>, + + publisher: Option<OptionalPublisher>, + + duration: Option<MaybeDuration>, + + url: Option<Url>, +} + +impl SelectCommand { + fn into_shared(self) -> Option<SharedSelectionCommandArgs> { + match self { + SelectCommand::File { .. } + | SelectCommand::Split { .. } + | SelectCommand::Add { .. } => None, + SelectCommand::Watch { shared } + | SelectCommand::Drop { shared } + | SelectCommand::Watched { shared } + | SelectCommand::Url { shared } + | SelectCommand::Pick { shared } => Some(shared), + } + } +} + +#[derive(Clone, Debug, Copy)] +struct OptionalNaiveDate { + date: Option<NaiveDate>, +} +impl FromStr for OptionalNaiveDate { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No release date]" { + Ok(Self { date: None }) + } else { + Ok(Self { + date: Some(NaiveDate::from_str(v)?), + }) + } + } +} +#[derive(Clone, Debug)] +struct OptionalPublisher { + publisher: Option<String>, +} +impl FromStr for OptionalPublisher { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No author]" { + Ok(Self { publisher: None }) + } else { + Ok(Self { + publisher: Some(v.to_owned()), + }) + } + } +} + +#[derive(Default, ValueEnum, Clone, Copy, Debug)] +enum SelectSplitSortKey { + /// Sort by the name of the publisher. + #[default] + Publisher, + + /// Sort by the number of unselected videos per publisher. + Videos, +} +impl Display for SelectSplitSortKey { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + SelectSplitSortKey::Publisher => f.write_str("publisher"), + SelectSplitSortKey::Videos => f.write_str("videos"), + } + } +} + +#[derive(Default, ValueEnum, Clone, Copy, Debug)] +enum SelectSplitSortMode { + /// Sort in ascending order (small -> big) + #[default] + Asc, + + /// Sort in descending order (big -> small) + Desc, +} + +impl Display for SelectSplitSortMode { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + SelectSplitSortMode::Asc => f.write_str("asc"), + SelectSplitSortMode::Desc => f.write_str("desc"), + } + } +} diff --git a/crates/yt/src/commands/show/implm/mod.rs b/crates/yt/src/commands/show/implm/mod.rs new file mode 100644 index 0000000..a2e40fd --- /dev/null +++ b/crates/yt/src/commands/show/implm/mod.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fs::{self, OpenOptions}, + io, + process::Command, +}; + +use crate::{ + app::App, + commands::ShowCommand, + output::{display_fmt_and_less, display_less}, + storage::db::video::Video, +}; + +use anyhow::{Context, Result, anyhow, bail}; +use tempfile::Builder; +use tokio_util::bytes::Buf; + +impl ShowCommand { + pub(in crate::commands) async fn implm(&self, app: &App) -> Result<()> { + match self { + ShowCommand::Description {} => { + let description = Video::get_current_description(app).await?; + + display_fmt_and_less(&description)?; + } + ShowCommand::Comments {} => { + let comments = Video::get_current_comments(app).await?; + + display_less(comments.render(app.config.global.display_colors))?; + } + ShowCommand::Thumbnail {} => { + let video = Video::currently_focused(app).await?.ok_or(anyhow!( + "You need to have a current video to display its info" + ))?; + + if let Some(url) = video.thumbnail_url { + let response = reqwest::get(url.clone()) + .await + .with_context(|| format!("Failed to download thumbnail from url: {url}"))?; + let response = response + .error_for_status() + .context("Failed to download thumbnail")?; + + let (tmp_path, mut tmp) = { + let file = Builder::new().prefix("yt-thumbnail-download").tempfile()?; + let (_, path) = file.keep()?; + let new_file = OpenOptions::new() + .write(true) + .read(false) + .create(false) + .truncate(true) + .open(&path)?; + + (path, new_file) + }; + + let mut content = response.bytes().await?.reader(); + io::copy(&mut content, &mut tmp)?; + + let status = Command::new(app.config.commands.image_show.first()) + .args(app.config.commands.image_show.tail()) + .arg(tmp_path.as_os_str()) + .status() + .context("Failed to spawn image show command")?; + + if !status.success() { + bail!( + "{:?} failed with status: {}", + &app.config.commands.image_show.join(" "), + status + ); + } + + fs::remove_file(&tmp_path).with_context(|| { + format!( + "Failed to cleanup downloaded thumbnail image at: {}", + tmp_path.display() + ) + })?; + } else { + eprintln!("Current video does not have a thumbnail."); + } + } + ShowCommand::Info {} => { + let video = Video::currently_focused(app).await?.ok_or(anyhow!( + "You need to have a current video to display its info" + ))?; + + display_less( + video + .to_info_display(app, None) + .await + .context("Failed to format video")?, + )?; + } + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/show/mod.rs b/crates/yt/src/commands/show/mod.rs new file mode 100644 index 0000000..60f2e51 --- /dev/null +++ b/crates/yt/src/commands/show/mod.rs @@ -0,0 +1,30 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Subcommand; + +mod implm; + +#[derive(Subcommand, Debug)] +pub(super) enum ShowCommand { + /// Display the description of the currently playing video + Description {}, + + /// Display the comments of the currently playing video. + Comments {}, + + /// Display the thumbnail of the currently playing video. + Thumbnail {}, + + /// Display general info of the currently playing video. + /// + /// This is the same as running `yt videos info <hash of current video>` + Info {}, +} diff --git a/crates/yt/src/commands/status/implm.rs b/crates/yt/src/commands/status/implm.rs new file mode 100644 index 0000000..dabc5df --- /dev/null +++ b/crates/yt/src/commands/status/implm.rs @@ -0,0 +1,157 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::time::Duration; + +use crate::{ + app::App, + commands::status::StatusCommand, + select::duration::MaybeDuration, + shared::bytes::Bytes, + storage::db::{ + subscription::Subscriptions, + video::{Video, VideoStatusMarker}, + }, + yt_dlp::get_current_cache_allocation, +}; + +use anyhow::{Context, Result}; + +macro_rules! get { + ($videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .count() + }; + + (@collect $videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .collect() + }; +} + +impl StatusCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let StatusCommand { format } = self; + + let all_videos = Video::in_states(app, VideoStatusMarker::ALL).await?; + + // lengths + let picked_videos_len = get!(all_videos, Pick); + + let watch_videos_len = get!(all_videos, Watch); + let cached_videos_len = get!(all_videos, Cached); + let watched_videos_len = get!(all_videos, Watched); + let watched_videos: Vec<_> = get!(@collect all_videos, Watched); + + let drop_videos_len = get!(all_videos, Drop); + let dropped_videos_len = get!(all_videos, Dropped); + + let subscriptions = Subscriptions::get(app).await?; + let subscriptions_len = subscriptions.0.len(); + + let watchtime_status = { + let total_watch_time_raw = watched_videos + .iter() + .fold(Duration::default(), |acc, vid| acc + vid.watch_progress); + + // Most things are watched at a speed of s (which is defined in the config file). + // Thus + // y = x * s -> y / s = x + let total_watch_time = Duration::from_secs_f64( + (total_watch_time_raw.as_secs_f64()) / app.config.select.playback_speed, + ); + + let speed = app.config.select.playback_speed; + + // Do not print the adjusted time, if the user has keep the speed level at 1. + #[allow(clippy::float_cmp)] + if speed == 1.0 { + format!( + "Total Watchtime: {}\n", + MaybeDuration::from_std(total_watch_time_raw) + ) + } else { + format!( + "Total Watchtime: {} (at {speed} speed: {})\n", + MaybeDuration::from_std(total_watch_time_raw), + MaybeDuration::from_std(total_watch_time), + ) + } + }; + + let watch_rate: f64 = { + fn to_f64(input: usize) -> f64 { + f64::from(u32::try_from(input).expect("This should never exceed u32::MAX")) + } + + let count = + to_f64(watched_videos_len) / (to_f64(drop_videos_len) + to_f64(dropped_videos_len)); + count * 100.0 + }; + + let cache_usage: Bytes = get_current_cache_allocation(app) + .await + .context("Failed to get current cache allocation")?; + + if let Some(fmt) = format { + let output = fmt + .replace( + "{picked_videos_len}", + picked_videos_len.to_string().as_str(), + ) + .replace("{watch_videos_len}", watch_videos_len.to_string().as_str()) + .replace( + "{cached_videos_len}", + cached_videos_len.to_string().as_str(), + ) + .replace( + "{watched_videos_len}", + watched_videos_len.to_string().as_str(), + ) + .replace("{watch_rate}", watch_rate.to_string().as_str()) + .replace("{drop_videos_len}", drop_videos_len.to_string().as_str()) + .replace( + "{dropped_videos_len}", + dropped_videos_len.to_string().as_str(), + ) + .replace("{watchtime_status}", watchtime_status.to_string().as_str()) + .replace( + "{subscriptions_len}", + subscriptions_len.to_string().as_str(), + ) + .replace("{cache_usage}", cache_usage.to_string().as_str()); + + print!("{output}"); + } else { + println!( + "\ +Picked Videos: {picked_videos_len} + +Watch Videos: {watch_videos_len} +Cached Videos: {cached_videos_len} +Watched Videos: {watched_videos_len} (watch rate: {watch_rate:.2} %) + +Drop Videos: {drop_videos_len} +Dropped Videos: {dropped_videos_len} + +{watchtime_status} + + Subscriptions: {subscriptions_len} + Cache usage: {cache_usage}" + ); + } + + Ok(()) + } +} diff --git a/crates/yt/src/commands/status/mod.rs b/crates/yt/src/commands/status/mod.rs new file mode 100644 index 0000000..4a8dee7 --- /dev/null +++ b/crates/yt/src/commands/status/mod.rs @@ -0,0 +1,20 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct StatusCommand { + /// Which format to use + #[arg(short, long)] + format: Option<String>, +} diff --git a/crates/yt/src/commands/subscriptions/implm.rs b/crates/yt/src/commands/subscriptions/implm.rs new file mode 100644 index 0000000..31b714e --- /dev/null +++ b/crates/yt/src/commands/subscriptions/implm.rs @@ -0,0 +1,253 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::str::FromStr; + +use crate::{ + app::App, + commands::subscriptions::SubscriptionCommand, + storage::db::{ + insert::{Operations, subscription::Operation}, + subscription::{Subscription, Subscriptions, check_url}, + }, +}; + +use anyhow::{Context, Result, bail}; +use log::{error, warn}; +use tokio::{ + fs::File, + io::{AsyncBufRead, AsyncBufReadExt, BufReader, stdin}, +}; +use url::Url; +use yt_dlp::{json_cast, json_get, json_try_get, options::YoutubeDLOptions}; + +impl SubscriptionCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + SubscriptionCommand::Add { + name, + url, + no_check, + } => { + let mut ops = Operations::new("main: subscribe"); + subscribe(app, name, url, no_check, &mut ops) + .await + .context("Failed to add a subscription")?; + ops.commit(app).await?; + } + SubscriptionCommand::Remove { name } => { + let mut present_subscriptions = Subscriptions::get(app).await?; + + let mut ops = Operations::new("Subscribe: unsubscribe"); + if let Some(subscription) = present_subscriptions.0.remove(&name) { + subscription.remove(&mut ops); + } else { + bail!("Couldn't find subscription: '{}'", &name); + } + ops.commit(app) + .await + .with_context(|| format!("Failed to unsubscribe from {name:?}"))?; + } + SubscriptionCommand::List {} => { + let all_subs = Subscriptions::get(app).await?; + + for (key, val) in all_subs.0 { + println!("{}: '{}'", key, val.url); + } + } + SubscriptionCommand::Export {} => { + let all_subs = Subscriptions::get(app).await?; + for val in all_subs.0.values() { + println!("{}", val.url); + } + } + SubscriptionCommand::Import { + file, + force, + no_check, + } => { + if let Some(file) = file { + let f = File::open(file).await?; + + import(app, BufReader::new(f), force, no_check).await?; + } else { + import(app, BufReader::new(stdin()), force, no_check).await?; + } + } + } + + Ok(()) + } +} + +async fn import<W: AsyncBufRead + AsyncBufReadExt + Unpin>( + app: &App, + reader: W, + force: bool, + no_check: bool, +) -> Result<()> { + let mut ops = Operations::new("SubscribeImport: init"); + + let all = Subscriptions::get(app).await?; + if force { + all.remove(&mut ops); + } + ops.commit(app).await?; + let mut ops = Operations::new("SubscribeImport: after all subs remove"); + + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + let url = + Url::from_str(&line).with_context(|| format!("Failed to parse '{line}' as url"))?; + + match subscribe(app, None, url, no_check, &mut ops) + .await + .with_context(|| format!("Failed to subscribe to: '{line}'")) + { + Ok(()) => (), + Err(err) => eprintln!( + "Error while subscribing to '{}': '{}'", + line, + err.source().expect("Should have a source") + ), + } + } + ops.commit(app).await?; + + Ok(()) +} + +async fn subscribe( + app: &App, + name: Option<String>, + url: Url, + no_check: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + if !(url.as_str().ends_with("videos") + || url.as_str().ends_with("streams") + || url.as_str().ends_with("shorts") + || url.as_str().ends_with("videos/") + || url.as_str().ends_with("streams/") + || url.as_str().ends_with("shorts/")) + && url.as_str().contains("youtube.com") + { + warn!( + "Your youtube url does not seem like it actually tracks a channels playlist \ + (videos, streams, shorts). Adding subscriptions for each of them..." + ); + + let url = Url::parse(&(url.as_str().to_owned() + "/")) + .expect("This was an url, it should stay one"); + + let (videos, streams, shorts) = if let Some(name) = name { + ( + Some(name.clone() + " {Videos}"), + Some(name.clone() + " {Streams}"), + Some(name.clone() + " {Shorts}"), + ) + } else { + (None, None, None) + }; + + let _ = actual_subscribe( + app, + videos, + url.join("videos/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!("Failed to subscribe to the '{}' variant: {err}", "{Videos}"); + }); + + let _ = actual_subscribe( + app, + streams, + url.join("streams/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!( + "Failed to subscribe to the '{}' variant: {err}", + "{Streams}" + ); + }); + + let _ = actual_subscribe( + app, + shorts, + url.join("shorts/").expect("See above."), + no_check, + ops, + ) + .await + .map_err(|err| { + error!("Failed to subscribe to the '{}' variant: {err}", "{Shorts}"); + }); + } else { + actual_subscribe(app, name, url, no_check, ops).await?; + } + + Ok(()) +} + +async fn actual_subscribe( + app: &App, + name: Option<String>, + url: Url, + no_check: bool, + ops: &mut Operations<Operation>, +) -> Result<()> { + if !no_check && !check_url(url.clone()).await? { + bail!("The url ('{}') does not represent a playlist!", &url) + } + + let name = if let Some(name) = name { + name + } else { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + if json_try_get!(info, "_type", as_str) == Some("playlist") { + json_get!(info, "title", as_str).to_owned() + } else { + bail!("The url ('{}') does not represent a playlist!", &url) + } + }; + + let present_subscriptions = Subscriptions::get(app).await?; + + if let Some(subs) = present_subscriptions.0.get(&name) { + bail!( + "The subscription '{}' could not be added, \ + as another one with the same name ('{}') already exists. \ + It points to the Url: '{}'", + name, + name, + subs.url + ); + } + + let sub = Subscription { name, url }; + + sub.add(ops); + + Ok(()) +} diff --git a/crates/yt/src/commands/subscriptions/mod.rs b/crates/yt/src/commands/subscriptions/mod.rs new file mode 100644 index 0000000..edd41c6 --- /dev/null +++ b/crates/yt/src/commands/subscriptions/mod.rs @@ -0,0 +1,62 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::PathBuf; + +use clap::Subcommand; +use clap_complete::ArgValueCompleter; +use url::Url; + +use crate::commands::complete_subscription; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +pub(super) enum SubscriptionCommand { + /// Subscribe to an URL + Add { + #[arg(short, long)] + /// The human readable name of the subscription + name: Option<String>, + + /// The URL to listen to + url: Url, + + /// Don't check, whether the URL actually points to something yt understands. + #[arg(long, default_value_t = false)] + no_check: bool, + }, + + /// Unsubscribe from an URL + Remove { + /// The human readable name of the subscription + #[arg(add = ArgValueCompleter::new(complete_subscription))] + name: String, + }, + + /// Import a bunch of URLs as subscriptions. + Import { + /// The file containing the URLs. Will use Stdin otherwise. + file: Option<PathBuf>, + + /// Remove any previous subscriptions + #[arg(short, long)] + force: bool, + + /// Don't check, whether the URLs actually point to something yt understands. + #[arg(long, default_value_t = false)] + no_check: bool, + }, + /// Write all subscriptions in an format understood by `import` + Export {}, + + /// List all subscriptions + List {}, +} diff --git a/crates/yt/src/commands/update/implm/mod.rs b/crates/yt/src/commands/update/implm/mod.rs new file mode 100644 index 0000000..53c7415 --- /dev/null +++ b/crates/yt/src/commands/update/implm/mod.rs @@ -0,0 +1,62 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::update::{UpdateCommand, implm::updater::Updater}, + storage::db::{ + extractor_hash::ExtractorHash, + subscription::{Subscription, Subscriptions}, + }, +}; + +use anyhow::{Result, bail}; + +mod updater; + +impl UpdateCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + let UpdateCommand { + max_backlog, + subscriptions: subscription_names_to_update, + } = self; + + let mut all_subs = Subscriptions::get(app).await?; + + let max_backlog = max_backlog.unwrap_or(app.config.update.max_backlog); + + let subs: Vec<Subscription> = if subscription_names_to_update.is_empty() { + all_subs.0.into_values().collect() + } else { + subscription_names_to_update + .into_iter() + .map(|sub| { + if let Some(val) = all_subs.0.remove(&sub) { + Ok(val) + } else { + bail!( + "Your specified subscription to update '{}' is not a subscription!", + sub + ) + } + }) + .collect::<Result<_>>()? + }; + + // We can get away with not having to re-fetch the hashes every time, as the returned video + // should not contain duplicates. + let hashes = ExtractorHash::get_all(app).await?; + + let updater = Updater::new(max_backlog, app.config.update.pool_size, hashes); + updater.update(app, subs).await?; + + Ok(()) + } +} diff --git a/crates/yt/src/commands/update/implm/updater.rs b/crates/yt/src/commands/update/implm/updater.rs new file mode 100644 index 0000000..2b96bf2 --- /dev/null +++ b/crates/yt/src/commands/update/implm/updater.rs @@ -0,0 +1,205 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use anyhow::{Context, Result}; +use futures::{StreamExt, future::join_all, stream}; +use log::{Level, debug, error, log_enabled}; +use tokio::io::{AsyncWriteExt, stderr}; +use tokio_util::task::LocalPoolHandle; +use yt_dlp::{ + info_json::InfoJson, json_cast, json_try_get, options::YoutubeDLOptions, process_ie_result, + python_error::PythonError, +}; + +use crate::{ + ansi_escape_codes, + app::App, + storage::db::{ + extractor_hash::ExtractorHash, insert::Operations, subscription::Subscription, video::Video, + }, + yt_dlp::yt_dlp_opts_updating, +}; + +pub(super) struct Updater { + max_backlog: usize, + hashes: Vec<ExtractorHash>, + pool: LocalPoolHandle, +} + +static REACHED_NUMBER: AtomicUsize = const { AtomicUsize::new(1) }; + +impl Updater { + pub(super) fn new(max_backlog: usize, max_threads: usize, hashes: Vec<ExtractorHash>) -> Self { + let pool = LocalPoolHandle::new(max_threads); + + Self { + max_backlog, + hashes, + pool, + } + } + + pub(super) async fn update(self, app: &App, subscriptions: Vec<Subscription>) -> Result<()> { + let total_number = subscriptions.len(); + + let mut stream = stream::iter(subscriptions) + .map(|sub| self.get_new_entries(sub, total_number)) + .buffer_unordered(app.config.update.futures); + + while let Some(output) = stream.next().await { + let mut entries = output?; + + if let Some(next) = entries.next() { + let (sub, entry) = next; + process_subscription(app, sub, entry).await?; + + join_all(entries.map(|(sub, entry)| process_subscription(app, sub, entry))) + .await + .into_iter() + .collect::<Result<(), _>>()?; + } + } + + Ok(()) + } + + #[allow(clippy::too_many_lines)] + async fn get_new_entries( + &self, + sub: Subscription, + total_number: usize, + ) -> Result<impl Iterator<Item = (Subscription, InfoJson)>> { + let max_backlog = self.max_backlog; + let hashes = self.hashes.clone(); + + let yt_dlp = yt_dlp_opts_updating(max_backlog)?; + + self.pool + .spawn_pinned(move || { + async move { + if !log_enabled!(Level::Debug) { + ansi_escape_codes::clear_whole_line(); + ansi_escape_codes::move_to_col(1); + eprint!( + "({}/{total_number}) Checking playlist {}...", + REACHED_NUMBER.fetch_add(1, Ordering::Relaxed), + sub.name + ); + ansi_escape_codes::move_to_col(1); + stderr().flush().await?; + } + + let info = yt_dlp + .extract_info(&sub.url, false, false) + .with_context(|| format!("Failed to get playlist '{}'.", sub.name))?; + + let empty = vec![]; + let entries = json_try_get!(info, "entries", as_array).unwrap_or(&empty); + + let valid_entries: Vec<(Subscription, InfoJson)> = entries + .iter() + .take(max_backlog) + .filter_map(|entry| -> Option<(Subscription, InfoJson)> { + let extractor_hash = + ExtractorHash::from_info_json(json_cast!(entry, as_object)); + + if hashes.contains(&extractor_hash) { + debug!( + "Skipping entry, as it is \ + already present: '{extractor_hash}'", + ); + None + } else { + Some((sub.clone(), json_cast!(entry, as_object).to_owned())) + } + }) + .collect(); + + Ok(valid_entries + .into_iter() + .map(|(sub, entry)| { + let inner_yt_dlp = YoutubeDLOptions::new() + .set("noplaylist", true) + .build() + .expect("Worked before, should work now"); + + match inner_yt_dlp.process_ie_result(entry, false) { + Ok(output) => Ok((sub, output)), + Err(err) => Err(err), + } + }) + // Don't fail the whole update, if one of the entries fails to fetch. + .filter_map(move |base| match base { + Ok(ok) => Some(ok), + Err(err) => { + match err { + process_ie_result::Error::Python(PythonError(err)) => { + if err.contains( + "Join this channel to get access \ + to members-only content ", + ) { + // Hide this error + } else { + // Show the error, but don't fail. + let error = err + .strip_prefix( + "DownloadError: \u{1b}[0;31mERROR:\u{1b}[0m ", + ) + .unwrap_or(&err); + error!("While fetching {:#?}: {error}", sub.name); + } + + None + } + process_ie_result::Error::InfoJsonPrepare(error) => { + error!( + "While fetching {:#?}: Failed to prepare \ + info json: {error}", + sub.name + ); + None + } + } + } + })) + } + }) + .await? + } +} + +async fn process_subscription(app: &App, sub: Subscription, entry: InfoJson) -> Result<()> { + let mut ops = Operations::new("Update: process subscription"); + let video = Video::from_info_json(&entry, Some(&sub)) + .context("Failed to parse search entry as Video")?; + + let title = video.title.clone(); + let url = video.url.clone(); + let video = video.add(&mut ops).with_context(|| { + format!("Failed to add video to database: '{title}' (with url: '{url}')") + })?; + + ops.commit(app).await.with_context(|| { + format!( + "Failed to add video to database: '{}' (with url: '{}')", + video.title, video.url + ) + })?; + println!( + "{}", + &video + .to_line_display(app, None) + .await + .with_context(|| format!("Failed to format video: '{}'", video.title))? + ); + Ok(()) +} diff --git a/crates/yt/src/commands/update/mod.rs b/crates/yt/src/commands/update/mod.rs new file mode 100644 index 0000000..cb29148 --- /dev/null +++ b/crates/yt/src/commands/update/mod.rs @@ -0,0 +1,27 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; +use clap_complete::ArgValueCompleter; + +use crate::commands::complete_subscription; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct UpdateCommand { + /// The maximal number of videos to fetch for each subscription. + #[arg(short, long)] + max_backlog: Option<usize>, + + /// The subscriptions to update + #[arg(add = ArgValueCompleter::new(complete_subscription))] + subscriptions: Vec<String>, +} diff --git a/crates/yt/src/commands/videos/implm.rs b/crates/yt/src/commands/videos/implm.rs new file mode 100644 index 0000000..2a018c7 --- /dev/null +++ b/crates/yt/src/commands/videos/implm.rs @@ -0,0 +1,73 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + commands::videos::VideosCommand, + storage::db::video::{Video, VideoStatusMarker}, +}; + +use anyhow::{Context, Result}; +use futures::{TryStreamExt, stream::FuturesUnordered}; + +impl VideosCommand { + pub(in crate::commands) async fn implm(self, app: &App) -> Result<()> { + match self { + VideosCommand::List { + search_query, + limit, + format, + } => { + let all_videos = Video::in_states(app, VideoStatusMarker::ALL).await?; + + // turn one video to a color display, to pre-warm the hash shrinking cache + if let Some(val) = all_videos.first() { + val.to_line_display(app, format.clone()).await?; + } + + let limit = limit.unwrap_or(all_videos.len()); + + let all_video_strings: Vec<String> = all_videos + .into_iter() + .take(limit) + .map(|vid| to_line_display_owned(vid, app, format.clone())) + .collect::<FuturesUnordered<_>>() + .try_collect::<Vec<String>>() + .await?; + + if let Some(query) = search_query { + all_video_strings + .into_iter() + .filter(|video| video.to_lowercase().contains(&query.to_lowercase())) + .for_each(|video| println!("{video}")); + } else { + println!("{}", all_video_strings.join("\n")); + } + } + VideosCommand::Info { hash, format } => { + let video = hash.realize(app, None).await?.get_with_app(app).await?; + + print!( + "{}", + &video + .to_info_display(app, format) + .await + .context("Failed to format video")? + ); + } + } + + Ok(()) + } +} + +async fn to_line_display_owned(video: Video, app: &App, format: Option<String>) -> Result<String> { + video.to_line_display(app, format).await +} diff --git a/crates/yt/src/commands/videos/mod.rs b/crates/yt/src/commands/videos/mod.rs new file mode 100644 index 0000000..ca20715 --- /dev/null +++ b/crates/yt/src/commands/videos/mod.rs @@ -0,0 +1,46 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::{ArgAction, Subcommand}; + +use crate::storage::db::extractor_hash::LazyExtractorHash; + +mod implm; + +#[derive(Subcommand, Clone, Debug)] +pub(super) enum VideosCommand { + /// List the videos in the database + #[command(visible_alias = "ls")] + List { + /// An optional search query to limit the results + #[arg(action = ArgAction::Append)] + search_query: Option<String>, + + /// The format string to use. + // TODO(@bpeetz): Encode the default format, as the default string here. <2025-07-04> + #[arg(short, long)] + format: Option<String>, + + /// The number of videos to show + #[arg(short, long)] + limit: Option<usize>, + }, + + /// Get detailed information about a video + Info { + /// The short hash of the video + hash: LazyExtractorHash, + + /// The format string to use. + // TODO(@bpeetz): Encode the default format, as the default string here. <2025-07-04> + #[arg(short, long)] + format: Option<String>, + }, +} diff --git a/crates/yt/src/commands/watch/implm/mod.rs b/crates/yt/src/commands/watch/implm/mod.rs new file mode 100644 index 0000000..8182216 --- /dev/null +++ b/crates/yt/src/commands/watch/implm/mod.rs @@ -0,0 +1,244 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fs, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use crate::{ + app::App, + commands::watch::{WatchCommand, implm::playlist_handler::Status}, + storage::{ + db::{ + insert::{Operations, maintenance::clear_stale_downloaded_paths}, + playlist::Playlist, + }, + notify::wait_for_db_write, + }, +}; + +use anyhow::{Context, Result}; +use libmpv2::{Mpv, events::EventContext}; +use log::{debug, info, trace, warn}; +use tokio::{task, time}; + +mod playlist_handler; + +impl WatchCommand { + #[allow(clippy::too_many_lines)] + pub(in crate::commands) async fn implm(self, app: Arc<App>) -> Result<()> { + let WatchCommand { + provide_ipc_socket, + headless, + } = self; + + clear_stale_downloaded_paths(&app).await?; + + let ipc_socket = if provide_ipc_socket { + Some(app.config.paths.mpv_ipc_socket_path.clone()) + } else { + None + }; + + let (mpv, mut ev_ctx) = + init_mpv(&app, ipc_socket, headless).context("Failed to initialize mpv instance")?; + let mpv = Arc::new(mpv); + + if provide_ipc_socket { + println!("{}", app.config.paths.mpv_ipc_socket_path.display()); + } + + let should_break = Arc::new(AtomicBool::new(false)); + let local_app = Arc::clone(&app); + let local_mpv = Arc::clone(&mpv); + let local_should_break = Arc::clone(&should_break); + let progress_handle = task::spawn(async move { + loop { + if local_should_break.load(Ordering::Relaxed) { + trace!("WatchProgressThread: Stopping, as we received exit signal."); + break; + } + + let mut playlist = Playlist::create(&local_app).await?; + + if let Some(index) = playlist.current_index() { + trace!("WatchProgressThread: Saving watch progress for current video"); + + let mut ops = + Operations::new("WatchProgressThread: save watch progress thread"); + playlist.save_watch_progress(&local_mpv, index, &mut ops); + ops.commit(&local_app).await?; + } else { + trace!( + "WatchProgressThread: Tried to save current watch progress, but no video active." + ); + } + + time::sleep(local_app.config.watch.progress_save_intervall).await; + } + + Ok::<(), anyhow::Error>(()) + }); + + let playlist = Playlist::create(&app).await?; + playlist.resync_with_mpv(&app, &mpv)?; + + let mut have_warned = (false, 0); + 'watchloop: loop { + 'waitloop: while let Ok(value) = playlist_handler::status(&app).await { + match value { + Status::NoMoreAvailable => { + break 'watchloop; + } + Status::NoCached { marked_watch } => { + // try again next time. + if have_warned.0 { + if have_warned.1 != marked_watch { + warn!("Now {marked_watch} videos are marked as to be watched."); + have_warned.1 = marked_watch; + } + } else { + warn!( + "There is nothing to watch yet, but still {marked_watch} videos marked as to be watched. \ + Will idle, until they become available" + ); + have_warned = (true, marked_watch); + } + wait_for_db_write(&app).await?; + + // Add the new videos, if they are there. + let playlist = Playlist::create(&app).await?; + playlist.resync_with_mpv(&app, &mpv)?; + } + Status::Available { newly_available } => { + debug!( + "Checked for currently available videos and found {newly_available}!" + ); + have_warned.0 = false; + + // Something just became available! + break 'waitloop; + } + } + } + + // TODO(@bpeetz): Is the following assumption correct? <2025-07-10> + // We wait until forever for the next event, because we really don't need to do anything + // else. + if let Some(ev) = ev_ctx.wait_event(f64::MAX) { + match ev { + Ok(event) => { + trace!("Mpv event triggered: {event:#?}"); + if playlist_handler::handle_mpv_event(&app, &mpv, &event) + .await + .with_context(|| format!("Failed to handle mpv event: '{event:#?}'"))? + { + break; + } + } + Err(e) => debug!("Mpv Event errored: {e}"), + } + } + } + should_break.store(true, Ordering::Relaxed); + progress_handle.await??; + + if provide_ipc_socket { + fs::remove_file(&app.config.paths.mpv_ipc_socket_path).with_context(|| { + format!( + "Failed to clean-up the mpv ipc socket at {}", + app.config.paths.mpv_ipc_socket_path.display() + ) + })?; + } + + Ok(()) + } +} + +fn init_mpv(app: &App, ipc_socket: Option<PathBuf>, headless: bool) -> Result<(Mpv, EventContext)> { + // set some default values, to make things easier (these can be overridden by the config file, + // which we load later) + let mpv = Mpv::with_initializer(|mpv| { + if let Some(socket) = ipc_socket { + mpv.set_property( + "input-ipc-server", + socket + .to_str() + .expect("This path comes from us, it should never contain not-utf8"), + )?; + } + + if headless { + // Do not provide video output. + mpv.set_property("vid", "no")?; + } else { + // Enable default key bindings, so the user can actually interact with + // the player (and e.g. close the window). + mpv.set_property("input-default-bindings", "yes")?; + mpv.set_property("input-vo-keyboard", "yes")?; + + // Show the on screen controller. + mpv.set_property("osc", "yes")?; + + // Don't automatically advance to the next video (or exit the player) + mpv.set_option("keep-open", "always")?; + + // Always display an window, even for non-video playback. + // As mpv does not have cli access, no window means no control and no user feedback. + mpv.set_option("force-window", "yes")?; + } + + Ok(()) + }) + .context("Failed to initialize mpv")?; + + let config_path = &app.config.paths.mpv_config_path; + if config_path.try_exists()? { + info!("Found mpv.conf at '{}'!", config_path.display()); + mpv.command( + "load-config-file", + &[config_path + .to_str() + .context("Failed to parse the config path is utf8-stringt")?], + )?; + } else { + warn!( + "Did not find a mpv.conf file at '{}'", + config_path.display() + ); + } + + let input_path = &app.config.paths.mpv_input_path; + if input_path.try_exists()? { + info!("Found mpv.input.conf at '{}'!", input_path.display()); + mpv.command( + "load-input-conf", + &[input_path + .to_str() + .context("Failed to parse the input path as utf8 string")?], + )?; + } else { + warn!( + "Did not find a mpv.input.conf file at '{}'", + input_path.display() + ); + } + + let ev_ctx = EventContext::new(mpv.ctx); + ev_ctx.disable_deprecated_events()?; + + Ok((mpv, ev_ctx)) +} diff --git a/crates/yt/src/commands/watch/implm/playlist_handler/client_messages.rs b/crates/yt/src/commands/watch/implm/playlist_handler/client_messages.rs new file mode 100644 index 0000000..fd7e035 --- /dev/null +++ b/crates/yt/src/commands/watch/implm/playlist_handler/client_messages.rs @@ -0,0 +1,93 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{env, time::Duration}; + +use crate::{app::App, storage::db::video::Video}; + +use anyhow::{Context, Result, bail}; +use libmpv2::Mpv; +use tokio::process::Command; + +use super::mpv_message; + +async fn run_self_in_external_command(app: &App, args: &[&str]) -> Result<()> { + // TODO(@bpeetz): Can we trust this value? <2025-06-15> + let binary = + env::current_exe().context("Failed to determine the current executable to re-execute")?; + + let arguments = [ + &[ + binary + .to_str() + .context("Failed to turn the executable path to a utf8-string")?, + "--db-path", + app.config + .paths + .database_path + .to_str() + .context("Failed to parse the database_path as a utf8-string")?, + ], + args, + ] + .concat(); + + let status = Command::new(app.config.commands.external_spawn.first()) + .args(app.config.commands.external_spawn.tail()) + .args(arguments) + .status() + .await?; + if !status.success() { + bail!("Falied to start (external) `yt {}`", args.join(" ")); + } + + Ok(()) +} + +pub(super) async fn handle_yt_description_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "description"]).await?; + Ok(()) +} +pub(super) async fn handle_yt_description_local(app: &App, mpv: &Mpv) -> Result<()> { + let description: String = Video::get_current_description(app) + .await? + .chars() + .take(app.config.watch.local_displays_length) + .collect(); + + mpv_message(mpv, &description, Duration::from_secs(6))?; + Ok(()) +} + +pub(super) async fn handle_yt_comments_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "comments"]).await?; + Ok(()) +} +pub(super) async fn handle_yt_comments_local(app: &App, mpv: &Mpv) -> Result<()> { + let comments: String = Video::get_current_comments(app) + .await? + .render(false) + .chars() + .take(app.config.watch.local_displays_length) + .collect(); + + mpv_message(mpv, &comments, Duration::from_secs(6))?; + Ok(()) +} + +pub(super) async fn handle_yt_info_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "info"]).await?; + Ok(()) +} + +pub(super) async fn handle_yt_thumbnail_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["show", "thumbnail"]).await?; + Ok(()) +} diff --git a/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs b/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs new file mode 100644 index 0000000..bdb77d2 --- /dev/null +++ b/crates/yt/src/commands/watch/implm/playlist_handler/mod.rs @@ -0,0 +1,225 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::time::Duration; + +use crate::{ + app::App, + storage::db::{ + insert::{Operations, playlist::VideoTransition}, + playlist::{Playlist, PlaylistIndex}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::{Context, Result}; +use libmpv2::{EndFileReason, Mpv, events::Event}; +use log::{debug, info}; + +mod client_messages; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum Status { + /// There are no videos cached and no more marked to be watched. + /// Waiting is pointless. + NoMoreAvailable, + + /// There are no videos cached, but some (> 0) are marked to be watched. + /// So we should wait for them to become available. + NoCached { marked_watch: usize }, + + /// There are videos cached and ready to be inserted into the playback queue. + Available { newly_available: usize }, +} + +fn mpv_message(mpv: &Mpv, message: &str, time: Duration) -> Result<()> { + mpv.command( + "show-text", + &[message, time.as_millis().to_string().as_str()], + )?; + Ok(()) +} + +/// Return the status of the playback queue +pub(crate) async fn status(app: &App) -> Result<Status> { + let playlist = Playlist::create(app).await?; + + let playlist_len = playlist.len(); + let marked_watch_num = Video::in_states(app, &[VideoStatusMarker::Watch]) + .await? + .len(); + + if playlist_len == 0 && marked_watch_num == 0 { + Ok(Status::NoMoreAvailable) + } else if playlist_len == 0 && marked_watch_num != 0 { + Ok(Status::NoCached { + marked_watch: marked_watch_num, + }) + } else if playlist_len != 0 { + Ok(Status::Available { + newly_available: playlist_len, + }) + } else { + unreachable!( + "The playlist length is {playlist_len}, but the number of marked watch videos is {marked_watch_num}! This is a bug." + ); + } +} + +/// # Returns +/// This will return [`true`], if the event handling should be stopped +/// +/// # Panics +/// Only if internal assertions fail. +#[allow(clippy::too_many_lines)] +pub(crate) async fn handle_mpv_event(app: &App, mpv: &Mpv, event: &Event<'_>) -> Result<bool> { + let mut ops = Operations::new("PlaylistHandler: handle event"); + + // Construct the playlist lazily. + // This avoids unneeded db lookups. + // (We use the moved `call_once` as guard for this) + let call_once = String::new(); + let playlist = move || { + drop(call_once); + Playlist::create(app) + }; + + let should_stop_event_handling = match event { + Event::EndFile(r) => match r.reason { + EndFileReason::Eof => { + info!("Mpv reached the end of the current video. Marking it watched."); + playlist().await?.resync_with_mpv(app, mpv)?; + + false + } + EndFileReason::Stop => { + // This reason is incredibly ambiguous. It _both_ means actually pausing a + // video and going to the next one in the playlist. + // Oh, and it's also called, when a video is removed from the playlist (at + // least via "playlist-remove current") + info!("Paused video (or went to next playlist entry); Doing nothing"); + + false + } + EndFileReason::Quit => { + info!("Mpv quit. Exiting playback"); + + playlist().await?.save_current_watch_progress(mpv, &mut ops); + + true + } + EndFileReason::Error => { + unreachable!("This should have been raised as a separate error") + } + EndFileReason::Redirect => { + // TODO: We probably need to handle this somehow <2025-02-17> + false + } + }, + Event::StartFile(_) => { + let mut playlist = playlist().await?; + + let mpv_pos = usize::try_from(mpv.get_property::<i64>("playlist-pos")?) + .expect("The value is strictly positive"); + + let yt_pos = playlist.current_index().map(usize::from); + + if (Some(mpv_pos) != yt_pos) || yt_pos.is_none() { + debug!( + "StartFileHandler: mpv pos {mpv_pos} and our pos {yt_pos:?} do not align. Reloading.." + ); + + if let Some((_, vid)) = playlist.get_focused_mut() { + vid.set_focused(false, &mut ops); + ops.commit(app) + .await + .context("Failed to commit video unfocusing")?; + + ops = Operations::new("PlaylistHandler: after set-focused"); + } + + let video = playlist + .get_mut(PlaylistIndex::from(mpv_pos)) + .expect("The mpv pos should not be out of bounds"); + + video.set_focused(true, &mut ops); + + playlist.resync_with_mpv(app, mpv)?; + } + + false + } + Event::Seek => { + playlist().await?.save_current_watch_progress(mpv, &mut ops); + + false + } + Event::ClientMessage(a) => { + debug!("Got Client Message event: '{}'", a.join(" ")); + + match a.as_slice() { + &["yt-comments-external"] => { + client_messages::handle_yt_comments_external(app).await?; + } + &["yt-comments-local"] => { + client_messages::handle_yt_comments_local(app, mpv).await?; + } + + &["yt-description-external"] => { + client_messages::handle_yt_description_external(app).await?; + } + &["yt-description-local"] => { + client_messages::handle_yt_description_local(app, mpv).await?; + } + + &["yt-info-external"] => { + client_messages::handle_yt_info_external(app).await?; + } + &["yt-thumbnail-external"] => { + client_messages::handle_yt_thumbnail_external(app).await?; + } + + &["yt-mark-picked"] => { + playlist().await?.mark_current_done( + app, + mpv, + VideoTransition::Picked, + &mut ops, + )?; + + mpv_message(mpv, "Marked the video as picked", Duration::from_secs(3))?; + } + &["yt-mark-watched"] => { + playlist().await?.mark_current_done( + app, + mpv, + VideoTransition::Watched, + &mut ops, + )?; + + mpv_message(mpv, "Marked the video watched", Duration::from_secs(3))?; + } + &["yt-check-new-videos"] => { + playlist().await?.resync_with_mpv(app, mpv)?; + } + other => { + debug!("Unknown message: {}", other.join(" ")); + } + } + + false + } + _ => false, + }; + + ops.commit(app).await?; + + Ok(should_stop_event_handling) +} diff --git a/crates/yt/src/commands/watch/mod.rs b/crates/yt/src/commands/watch/mod.rs new file mode 100644 index 0000000..ea4c513 --- /dev/null +++ b/crates/yt/src/commands/watch/mod.rs @@ -0,0 +1,24 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use clap::Parser; + +mod implm; + +#[derive(Parser, Debug)] +pub(super) struct WatchCommand { + /// Print the path to an ipc socket for mpv control to stdout at startup. + #[arg(long)] + provide_ipc_socket: bool, + + /// Don't start an mpv window at all. + #[arg(long)] + headless: bool, +} diff --git a/crates/yt/src/config/mod.rs b/crates/yt/src/config/mod.rs new file mode 100644 index 0000000..05bb4cf --- /dev/null +++ b/crates/yt/src/config/mod.rs @@ -0,0 +1,138 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use crate::config::support::mk_config; + +mod non_empty_vec; +mod paths; +mod support; + +pub(crate) static SHOULD_DISPLAY_COLOR: AtomicBool = AtomicBool::new(false); + +// We need to do both things to comply with what the config expects. +#[allow(clippy::trivially_copy_pass_by_ref, clippy::unnecessary_wraps)] +fn set_static_should_display_color(value: &bool) -> anyhow::Result<()> { + SHOULD_DISPLAY_COLOR.store(*value, Ordering::Relaxed); + + Ok(()) +} + +mk_config! { + use std::path::PathBuf; + use std::io::IsTerminal; + use std::time::Duration; + + use crate::shared::bytes::Bytes; + + use super::set_static_should_display_color; + + use super::paths::get_config_path; + use super::paths::get_runtime_path; + use super::paths::get_data_path; + use super::paths::ensure_parent_dir; + use super::paths::ensure_dir; + use super::paths::PREFIX; + + use super::non_empty_vec::NonEmptyVec; + use super::non_empty_vec::non_empty_vec; + + struct Config { + global: GlobalConfig = { + /// Whether to display colors. + display_colors: bool where display_color: Option<bool> =! {|config_value: Option<bool>| + Ok::<_, anyhow::Error>( + display_color + .unwrap_or( + config_value + .unwrap_or_else(|| std::io::stderr().is_terminal()) + ) + ) + } => set_static_should_display_color, + }, + select: SelectConfig = { + /// The playback speed to use, when it is not overridden. + playback_speed: f64 =: 2.7, + + /// The subtitle langs to download, when it is not overridden. + subtitle_langs: String =: String::new(), + }, + watch: WatchConfig = { + /// How many chars to display at most, when displaying information on mpv's local on screen + /// display. + local_displays_length: usize =: 1000, + + /// How long to wait between saving the video watch progress. + progress_save_intervall: Duration =: Duration::from_secs(10), + }, + commands: CommandsConfig = { + /// Which command to execute, when showing the thumbnail. + /// + /// This command will be executed with the one argument, being the path to the image file to display. + image_show: NonEmptyVec<String> =: non_empty_vec!["imv".to_owned()], + + /// Which command to use, when spawing one of the external commands (e.g. + /// `yt-comments-external` from mpv). + /// + /// The command will be called with a series of args that should be executed. + /// For example, + /// `<your_specified_command> <path_to_yt_binary> --db-path <path_to_current_db_path> comments` + external_spawn: NonEmptyVec<String> =: non_empty_vec!["alacritty".to_owned(), "-e".to_owned()], + + /// Which command to use, when opening video urls (like in the `yt select url` case). + /// + /// This command will be called with one argument, being the url of the video to open. + url_opener: NonEmptyVec<String> =: non_empty_vec!["firefox".to_owned()], + }, + paths: PathsConfig = { + /// Where to store downloaded files. + download_dir: PathBuf =: { + // We download to the temp dir to avoid taxing the disk + let temp_dir = std::env::temp_dir(); + + temp_dir.join(PREFIX) + } => ensure_dir, + + /// Path to the mpv configuration file. + mpv_config_path: PathBuf =? get_config_path("mpv.conf") => ensure_parent_dir, + + /// Path to the mpv input configuration file. + mpv_input_path: PathBuf =? get_config_path("mpv.input.conf") => ensure_parent_dir, + + /// Which path to use for mpv ipc socket creation. + mpv_ipc_socket_path: PathBuf =? get_runtime_path("mpv.ipc.socket") => ensure_parent_dir, + + /// Path to the video database. + database_path: PathBuf where db_path: Option<PathBuf> =! {|config_value: Option<PathBuf>| { + db_path.map_or_else(|| config_value.map_or_else(|| get_data_path("videos.sqlite"), Ok), Ok) + }} => ensure_parent_dir, + + /// Where to store the selection file before applying it. + last_selection_path: PathBuf =? get_runtime_path("selected.yts") => ensure_parent_dir, + }, + download: DownloadConfig = { + /// The maximum cache size. + max_cache_size: Bytes =? "3 GiB".parse(), + }, + update: UpdateConfig = { + /// How many videos to download, when checking for new ones. + max_backlog: usize =: 20, + + /// How many threads to use in the thread pool for fetching new videos. + pool_size: usize =: 16, + + /// How many subscriptions to fetch at once. + /// + /// For example, 16 means, that we will fetch 16 subscriptions at the same time. + futures: usize =: 16 * 4, + }, + } +} diff --git a/crates/yt/src/config/non_empty_vec.rs b/crates/yt/src/config/non_empty_vec.rs new file mode 100644 index 0000000..bd2c5e3 --- /dev/null +++ b/crates/yt/src/config/non_empty_vec.rs @@ -0,0 +1,83 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::VecDeque, + fmt::{Display, Write}, +}; + +use anyhow::bail; +use serde::{Deserialize, Serialize}; + +macro_rules! non_empty_vec { + ($first:expr $(, $($others:expr),+ $(,)?)?) => {{ + let inner: Vec<_> = vec![$first $(, $($others,)+)?]; + inner.try_into().expect("Has a first arg") + }} +} +pub(crate) use non_empty_vec; + +/// A vector that is non-empty. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(try_from = "Vec<T>")] +#[serde(into = "Vec<T>")] +pub(crate) struct NonEmptyVec<T: Clone> { + first: T, + rest: Vec<T>, +} + +impl<T: Clone> TryFrom<Vec<T>> for NonEmptyVec<T> { + type Error = anyhow::Error; + + fn try_from(value: Vec<T>) -> Result<Self, Self::Error> { + let mut queue = VecDeque::from(value); + + if let Some(first) = queue.pop_front() { + Ok(Self { + first, + rest: queue.into(), + }) + } else { + bail!("You need to have at least one element in a non-empty vector.") + } + } +} + +impl<T: Clone> From<NonEmptyVec<T>> for Vec<T> { + fn from(value: NonEmptyVec<T>) -> Self { + let mut base = vec![value.first]; + base.extend(value.rest); + base + } +} + +impl<T: Clone> NonEmptyVec<T> { + pub(crate) fn first(&self) -> &T { + &self.first + } + + pub(crate) fn tail(&self) -> &[T] { + self.rest.as_ref() + } + + pub(crate) fn join(&self, sep: &str) -> String + where + T: Display, + { + let mut output = String::new(); + write!(output, "{}", self.first()).expect("In-memory, does not fail"); + + for elem in &self.rest { + write!(output, "{sep}{elem}").expect("In-memory, does not fail"); + } + + output + } +} diff --git a/crates/yt/src/config/paths.rs b/crates/yt/src/config/paths.rs new file mode 100644 index 0000000..66975dd --- /dev/null +++ b/crates/yt/src/config/paths.rs @@ -0,0 +1,58 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +pub(super) fn get_runtime_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_runtime_file(name) + .with_context(|| format!("Failed to place runtime file: '{name}'")) +} +pub(super) fn get_data_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_data_file(name) + .with_context(|| format!("Failed to place data file: '{name}'")) +} +pub(super) fn get_config_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_config_file(name) + .with_context(|| format!("Failed to place config file: '{name}'")) +} + +pub(super) fn ensure_parent_dir(path: &Path) -> Result<()> { + if !path.exists() { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + } + + Ok(()) +} +pub(super) fn ensure_dir(path: &Path) -> Result<()> { + if !path.exists() { + std::fs::create_dir_all(path) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + + Ok(()) +} + +pub(super) fn config_path() -> Result<PathBuf> { + get_config_path("config.toml") +} + +pub(crate) const PREFIX: &str = "yt"; diff --git a/crates/yt/src/config/support.rs b/crates/yt/src/config/support.rs new file mode 100644 index 0000000..96e7ba4 --- /dev/null +++ b/crates/yt/src/config/support.rs @@ -0,0 +1,161 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +macro_rules! mk_config { + ( + $(use $usage_path:path;)* + + struct $name:ident { + $( + $(#[$attr0:meta])* + $subconfig_name:ident : $subconfig_type:ident = { + $( + $(#[$attr1:meta])* + $field_name:ident : $field_type:ty $( + where $extra_input:ident: $extra_input_type:ty + ),* = $errors:tt $default:expr $(=> $finalizer:ident)? + ),* + $(,)? + } + ),* + $(,)? + } + ) => { + mod _inner { + #![allow(non_snake_case)] + + $(use $usage_path;)* + + #[derive(serde::Serialize, Debug)] + pub(crate) struct $name { + $( + $(#[$attr0])* + pub(crate) $subconfig_name: $subconfig_type + ),* + } + + #[derive(Debug, serde::Deserialize, PartialEq)] + #[serde(deny_unknown_fields)] + #[allow(non_camel_case_types)] + struct config { + $( + $subconfig_name: Option<$subconfig_name> + ),* + } + + impl $name { + pub(crate) fn from_config_file( + config_file_path: Option<std::path::PathBuf>, + $( + $( + $( + $extra_input: $extra_input_type, + )* + )* + )* + ) -> anyhow::Result<Self> { + use anyhow::Context; + + let config_file_path = + config_file_path.map_or_else(|| -> anyhow::Result<_> { super::paths::config_path() }, Ok)?; + + let config: config = + toml::from_str(&std::fs::read_to_string(config_file_path).unwrap_or(String::new())) + .context("Failed to parse the config file as toml")?; + + Ok(Self { + $( + $subconfig_name: { + let toplevel = config.$subconfig_name.unwrap_or_default(); + $subconfig_type { + $( + $field_name: $field_name(toplevel.$field_name, $($extra_input),*)? + ),* + } + } + ),* + }) + } + + pub(crate) fn run_finalizers(&self) -> anyhow::Result<()> { + #[allow(unused_imports)] + use anyhow::Context; + + $( + $( + $( + $finalizer(&self.$subconfig_name.$field_name) + .context( + concat!( + "While running the finalizer for config value '", + stringify!($subconfig_name), + ".", + stringify!($field_name), + "'" + ) + )?; + )? + )* + )* + + Ok(()) + } + } + + $( + #[derive(serde::Serialize, Debug)] + pub(crate) struct $subconfig_type { + $( + $(#[$attr1])* + pub(crate) $field_name: $field_type + ),* + } + + #[derive(Debug, Default, serde::Deserialize, PartialEq)] + #[serde(deny_unknown_fields)] + #[allow(non_camel_case_types)] + struct $subconfig_name { + $( + $field_name: Option<$field_type> + ),* + } + + $( + fn $field_name( + config_value: Option<$field_type>, + $($extra_input: $extra_input_type),* + ) -> anyhow::Result<$field_type> { + use anyhow::Context; + + let expr = $crate::config::support::maybe_wrap_type!($field_type =$errors $default)(config_value); + + expr.context(concat!("Failed to generate default config value for '", stringify!($field_name),"'")) + } + )* + )* + } + pub(crate) use self::_inner::*; + }; +} + +macro_rules! maybe_wrap_type { + ($ty:ty =! $val:expr) => { + (|config_value: Option<$ty>| $val(config_value)) + }; + ($ty:ty =? $val:expr) => { + (|config_value: Option<$ty>| config_value.map_or_else(|| $val, Ok)) + }; + ($ty:ty =: $val:expr) => { + (|config_value: Option<$ty>| Ok::<_, anyhow::Error>(config_value.unwrap_or_else(|| $val))) + }; +} + +pub(crate) use maybe_wrap_type; +pub(crate) use mk_config; diff --git a/crates/yt/src/main.rs b/crates/yt/src/main.rs new file mode 100644 index 0000000..705e642 --- /dev/null +++ b/crates/yt/src/main.rs @@ -0,0 +1,89 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// `yt` is not a library. Besides, the `anyhow::Result` type is really useless, if you're not going +// to print it anyways. +#![allow(clippy::missing_errors_doc, clippy::missing_panics_doc)] + +use anyhow::{Context, Result}; +use app::App; +use clap::{CommandFactory, Parser}; +use config::Config; +use log::info; + +use crate::commands::Command; + +pub(crate) mod output; +pub(crate) mod yt_dlp; + +pub(crate) mod ansi_escape_codes; +pub(crate) mod app; +pub(crate) mod cli; +pub(crate) mod commands; +pub(crate) mod shared; + +pub(crate) mod config; +pub(crate) mod select; +pub(crate) mod storage; +pub(crate) mod version; +pub(crate) mod videos; + +#[tokio::main] +async fn main() -> Result<()> { + clap_complete::CompleteEnv::with_factory(cli::CliArgs::command).complete(); + + let args = cli::CliArgs::parse(); + + // The default verbosity is 1 (Warn) + let verbosity: u8 = args.verbosity + 1; + + stderrlog::new() + .module(module_path!()) + .modules(&["yt_dlp".to_owned(), "libmpv2".to_owned()]) + .quiet(args.quiet) + .show_module_names(false) + .color(stderrlog::ColorChoice::Auto) + .verbosity(verbosity as usize) + .timestamp(stderrlog::Timestamp::Off) + .init() + .expect("Let's just hope that this does not panic"); + + info!("Using verbosity level: '{} ({})'", verbosity, { + match verbosity { + 0 => "Error", + 1 => "Warn", + 2 => "Info", + 3 => "Debug", + 4.. => "Trace", + } + }); + + let config = Config::from_config_file(args.config_path, args.color, args.db_path)?; + if args.version { + version::show(&config).await?; + return Ok(()); + } + + // Perform config finalization _after_ checking for the version + // so that version always works. + config + .run_finalizers() + .context("Failed to finalize config for usage")?; + + let app = App::new(config, !args.no_migrate_db).await?; + + args.command + .unwrap_or(Command::default()) + .implm(app) + .await?; + + Ok(()) +} diff --git a/crates/yt/src/output/mod.rs b/crates/yt/src/output/mod.rs new file mode 100644 index 0000000..2f74519 --- /dev/null +++ b/crates/yt/src/output/mod.rs @@ -0,0 +1,56 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + io::Write, + process::{Command, Stdio}, +}; + +use anyhow::{Context, Result}; +use uu_fmt::{FmtOptions, process_text}; + +pub(crate) fn display_less(input: String) -> Result<()> { + let mut less = Command::new("less") + .args(["--raw-control-chars"]) + .stdin(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .context("Failed to run less")?; + + let mut stdin = less.stdin.take().context("Failed to open stdin")?; + std::thread::spawn(move || { + stdin + .write_all(input.as_bytes()) + .expect("Should be able to write to the stdin of less"); + }); + + let _ = less.wait().context("Failed to await less")?; + + Ok(()) +} + +pub(crate) fn display_fmt_and_less(input: &str) -> Result<()> { + display_less(format_text(&input, None)) +} + +#[must_use] +pub(crate) fn format_text(input: &str, termsize: Option<u16>) -> String { + let input = input.trim(); + + let width = termsize.unwrap_or_else(|| termsize::get().map_or(90, |size| size.cols)); + let fmt_opts = FmtOptions { + uniform: true, + split_only: true, + ..FmtOptions::new(Some(width as usize), None, Some(4)) + }; + + process_text(input, &fmt_opts) +} diff --git a/crates/yt/src/select/duration.rs b/crates/yt/src/select/duration.rs new file mode 100644 index 0000000..f1de2ea --- /dev/null +++ b/crates/yt/src/select/duration.rs @@ -0,0 +1,240 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::str::FromStr; +use std::time::Duration; + +use anyhow::{Result, bail}; + +const SECOND: u64 = 1; +const MINUTE: u64 = 60 * SECOND; +const HOUR: u64 = 60 * MINUTE; +const DAY: u64 = 24 * HOUR; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct MaybeDuration { + time: Option<Duration>, +} + +impl MaybeDuration { + #[must_use] + pub(crate) fn from_std(d: Duration) -> Self { + Self { time: Some(d) } + } + + #[must_use] + pub(crate) fn from_secs_f64(d: f64) -> Self { + Self { + time: Some(Duration::from_secs_f64(d)), + } + } + #[must_use] + pub(crate) fn from_maybe_secs_f64(d: Option<f64>) -> Self { + Self { + time: d.map(Duration::from_secs_f64), + } + } + #[must_use] + #[cfg(test)] + pub(crate) fn from_secs(d: u64) -> Self { + Self { + time: Some(Duration::from_secs(d)), + } + } + + /// Try to return the current duration encoded as seconds. + #[must_use] + pub(crate) fn as_secs(&self) -> Option<u64> { + self.time.map(|v| v.as_secs()) + } + + /// Try to return the current duration encoded as seconds and nanoseconds. + #[must_use] + pub(crate) fn as_secs_f64(&self) -> Option<f64> { + self.time.map(|v| v.as_secs_f64()) + } +} + +impl FromStr for MaybeDuration { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + #[derive(Debug, Clone, Copy)] + enum Token { + Number(u64), + UnitConstant((char, u64)), + } + + struct Tokenizer<'a> { + input: &'a str, + } + + impl Tokenizer<'_> { + fn next(&mut self) -> Result<Option<Token>> { + loop { + if let Some(next) = self.peek() { + match next { + '0'..='9' => { + let mut number = self.expect_num(); + while matches!(self.peek(), Some('0'..='9')) { + number *= 10; + number += self.expect_num(); + } + break Ok(Some(Token::Number(number))); + } + 's' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('s', SECOND)))); + } + 'm' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('m', MINUTE)))); + } + 'h' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('h', HOUR)))); + } + 'd' => { + self.chomp(); + break Ok(Some(Token::UnitConstant(('d', DAY)))); + } + ' ' => { + // Simply ignore white space + self.chomp(); + } + other => bail!("Unknown unit: {other:#?}"), + } + } else { + break Ok(None); + } + } + } + + fn chomp(&mut self) { + self.input = &self.input[1..]; + } + + fn peek(&self) -> Option<char> { + self.input.chars().next() + } + + fn expect_num(&mut self) -> u64 { + let next = self.peek().expect("Should be some at this point"); + self.chomp(); + assert!(next.is_ascii_digit()); + (next as u64) - ('0' as u64) + } + } + + if s == "[No duration]" { + return Ok(Self { time: None }); + } + + let mut tokenizer = Tokenizer { input: s }; + + let mut value = 0; + let mut current_val = None; + while let Some(token) = tokenizer.next()? { + match token { + Token::Number(number) => { + if let Some(current_val) = current_val { + bail!("Failed to find unit for number: {current_val}"); + } + + { + current_val = Some(number); + } + } + Token::UnitConstant((name, unit)) => { + if let Some(cval) = current_val { + value += cval * unit; + current_val = None; + } else { + bail!("Found unit without number: {name:#?}"); + } + } + } + } + + if let Some(current_val) = current_val { + bail!("Duration endet without unit, number was: {current_val}"); + } + + Ok(Self { + time: Some(Duration::from_secs(value)), + }) + } +} + +impl std::fmt::Display for MaybeDuration { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + if let Some(self_seconds) = self.as_secs() { + let base_day = self_seconds - (self_seconds % DAY); + let base_hour = (self_seconds % DAY) - ((self_seconds % DAY) % HOUR); + let base_min = (self_seconds % HOUR) - (((self_seconds % DAY) % HOUR) % MINUTE); + let base_sec = ((self_seconds % DAY) % HOUR) % MINUTE; + + let d = base_day / DAY; + let h = base_hour / HOUR; + let m = base_min / MINUTE; + let s = base_sec / SECOND; + + if d > 0 { + write!(fmt, "{d}d {h}h {m}m") + } else if h > 0 { + write!(fmt, "{h}h {m}m") + } else { + write!(fmt, "{m}m {s}s") + } + } else { + write!(fmt, "[No duration]") + } + } +} +#[cfg(test)] +mod test { + use std::str::FromStr; + + use crate::select::duration::{DAY, HOUR, MINUTE}; + + use super::MaybeDuration; + + fn mk_roundtrip(input: MaybeDuration, expected: &str) { + let output = MaybeDuration::from_str(expected).unwrap(); + + assert_eq!(input.to_string(), output.to_string()); + assert_eq!(input.to_string(), expected); + assert_eq!( + MaybeDuration::from_str(input.to_string().as_str()).unwrap(), + output + ); + } + + #[test] + fn test_roundtrip_duration_1h() { + mk_roundtrip(MaybeDuration::from_secs(HOUR), "1h 0m"); + } + #[test] + fn test_roundtrip_duration_30min() { + mk_roundtrip(MaybeDuration::from_secs(MINUTE * 30), "30m 0s"); + } + #[test] + fn test_roundtrip_duration_1d() { + mk_roundtrip( + MaybeDuration::from_secs(DAY + MINUTE * 30 + HOUR * 2), + "1d 2h 30m", + ); + } + #[test] + fn test_roundtrip_duration_none() { + mk_roundtrip(MaybeDuration::from_maybe_secs_f64(None), "[No duration]"); + } +} diff --git a/crates/yt/src/select/mod.rs b/crates/yt/src/select/mod.rs new file mode 100644 index 0000000..b02677f --- /dev/null +++ b/crates/yt/src/select/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod duration; + +// // FIXME: There should be no reason why we need to re-run yt, just to get the help string. But I've +// // yet to find a way to do it without the extra exec <2024-08-20> +// async fn get_help() -> Result<String> { +// let binary_name = current_exe()?; +// let cmd = Command::new(binary_name) +// .args(&["select", "--help"]) +// .output() +// .await?; +// +// assert_eq!(cmd.status.code(), Some(0)); +// +// let output = String::from_utf8(cmd.stdout).expect("Our help output was not utf8?"); +// +// let out = output +// .lines() +// .map(|line| format!("# {}\n", line)) +// .collect::<String>(); +// +// debug!("Returning help: '{}'", &out); +// +// Ok(out) +// } diff --git a/crates/bytes/src/error.rs b/crates/yt/src/shared/bytes/error.rs index c9783d8..c9783d8 100644 --- a/crates/bytes/src/error.rs +++ b/crates/yt/src/shared/bytes/error.rs diff --git a/crates/bytes/src/lib.rs b/crates/yt/src/shared/bytes/mod.rs index 2a9248d..31e782e 100644 --- a/crates/bytes/src/lib.rs +++ b/crates/yt/src/shared/bytes/mod.rs @@ -16,6 +16,7 @@ )] use std::{fmt::Display, str::FromStr}; +use ::serde::{Deserialize, Serialize}; use error::BytesError; const B: u64 = 1; @@ -31,10 +32,11 @@ const MB: u64 = 1000 * KB; const GB: u64 = 1000 * MB; const TB: u64 = 1000 * GB; -pub mod error; -pub mod serde; +pub(crate) mod error; -#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd, Ord, Eq, Deserialize, Serialize)] +#[serde(try_from = "String")] +#[serde(into = "String")] pub struct Bytes(u64); impl Bytes { @@ -131,6 +133,20 @@ impl Display for Bytes { } } +impl From<Bytes> for String { + fn from(value: Bytes) -> Self { + value.to_string() + } +} + +impl TryFrom<String> for Bytes { + type Error = BytesError; + + fn try_from(value: String) -> Result<Self, Self::Error> { + value.as_str().parse() + } +} + // taken from this stack overflow question: https://stackoverflow.com/a/76572321 /// Round to significant digits (rather than digits after the decimal). /// @@ -149,7 +165,7 @@ impl Display for Bytes { ///# } /// ``` #[must_use] -pub fn precision_f64(x: f64, decimals: u32) -> f64 { +pub(crate) fn precision_f64(x: f64, decimals: u32) -> f64 { if x == 0. || decimals == 0 { 0. } else { diff --git a/crates/yt/src/shared/mod.rs b/crates/yt/src/shared/mod.rs new file mode 100644 index 0000000..d3cc563 --- /dev/null +++ b/crates/yt/src/shared/mod.rs @@ -0,0 +1,11 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod bytes; diff --git a/crates/yt/src/storage/db/extractor_hash.rs b/crates/yt/src/storage/db/extractor_hash.rs new file mode 100644 index 0000000..3ad8273 --- /dev/null +++ b/crates/yt/src/storage/db/extractor_hash.rs @@ -0,0 +1,220 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashSet, fmt::Display, str::FromStr}; + +use anyhow::{Context, Result, bail}; +use blake3::Hash; +use log::debug; +use serde::{Deserialize, Serialize}; +use tokio::sync::OnceCell; +use yt_dlp::{info_json::InfoJson, json_cast, json_get, json_try_get}; + +use crate::app::App; + +static EXTRACTOR_HASH_LENGTH: OnceCell<usize> = OnceCell::const_new(); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy, Serialize, Deserialize)] +pub(crate) struct ExtractorHash { + hash: Hash, +} + +impl Display for ExtractorHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.hash.fmt(f) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ShortHash(String); + +impl Display for ShortHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone)] +#[allow(clippy::module_name_repetitions)] +pub(crate) struct LazyExtractorHash { + value: ShortHash, +} + +impl FromStr for LazyExtractorHash { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { + // perform some cheap validation + if s.len() > 64 { + bail!("A hash can only contain 64 bytes!"); + } + + Ok(Self { + value: ShortHash(s.to_owned()), + }) + } +} + +impl LazyExtractorHash { + /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] + pub(crate) async fn realize( + self, + app: &App, + all_hashes: Option<&[ExtractorHash]>, + ) -> Result<ExtractorHash> { + ExtractorHash::from_short_hash(app, &self.value, all_hashes).await + } +} + +impl ExtractorHash { + #[must_use] + pub(crate) fn from_hash(hash: Hash) -> Self { + Self { hash } + } + + pub(crate) async fn from_short_hash( + app: &App, + s: &ShortHash, + all_hashes: Option<&[Self]>, + ) -> Result<Self> { + let all_hashes = if let Some(all) = all_hashes { + all + } else { + &Self::get_all(app) + .await + .context("Failed to fetch all extractor-hashes from the database")? + }; + let needed_chars = s.0.len(); + for hash in all_hashes { + // PERFORMANCE(@bpeetz): This could avoid the string construction and just use a + // numeric equality check instead. <2025-07-15> + if hash.hash().to_hex()[..needed_chars] == s.0 { + return Ok(*hash); + } + } + bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); + } + + pub(crate) fn from_info_json(entry: &InfoJson) -> Self { + // HACK(@bpeetz): The code that follows is a gross hack. + // One would expect the `id` to be unique _and_ constant for each and every possible info JSON. + // But .. it's just not. The `ARDMediathek` extractor, will sometimes return different `id`s for the same + // video, effectively causing us to insert the same video again into the db (which fails, + // because the URL is still unique). + // + // As such we _should_ probably find a constant value for all extractors, but that just does + // not exist currently, without processing each entry (which is expensive and which I would + // like to avoid). + // + // Therefor, we simply special case the `ARDBetaMediathek` extractor. <2025-07-04> + + // NOTE(@bpeetz): `yt-dlp` apparently uses these two different names for the same thing <2025-07-04> + let ie_key = { + if let Some(ie_key) = json_try_get!(entry, "ie_key", as_str) { + ie_key + } else if let Some(extractor_key) = json_try_get!(entry, "extractor_key", as_str) { + extractor_key + } else { + unreachable!( + "Either `ie_key` or `extractor_key` \ + should be set on every entry info json" + ) + } + }; + + if ie_key == "ARDBetaMediathek" { + // NOTE(@bpeetz): The mediathek is changing their Id scheme, from an `short` old Id to the + // new id. As the new id is too long for some people, yt-dlp will be default return the old + // one (when it is still available!). The new one is called `display_id`. + // Therefore, we simply check if the new one is explicitly returned, and otherwise use the + // normal `id` value, as these are cases where the old one is no longer available. <2025-07-04> + let id = if let Some(display_id) = json_try_get!(entry, "display_id", as_str) { + display_id.as_bytes() + } else { + json_get!(entry, "id", as_str).as_bytes() + }; + + Self { + hash: blake3::hash(id), + } + } else { + Self { + hash: blake3::hash(json_get!(entry, "id", as_str).as_bytes()), + } + } + } + + #[must_use] + pub(crate) fn hash(&self) -> &Hash { + &self.hash + } + + pub(crate) async fn as_short_hash(&self, app: &App) -> Result<ShortHash> { + let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { + *needed_chars + } else { + let needed_chars = self + .get_needed_char_len(app) + .await + .context("Failed to calculate needed char length")?; + EXTRACTOR_HASH_LENGTH + .set(needed_chars) + .expect("This should work at this stage, as we checked above that it is empty."); + + needed_chars + }; + + Ok(ShortHash( + self.hash() + .to_hex() + .chars() + .take(needed_chars) + .collect::<String>(), + )) + } + + async fn get_needed_char_len(&self, app: &App) -> Result<usize> { + debug!("Calculating the needed hash char length"); + let all_hashes = Self::get_all(app) + .await + .context("Failed to fetch all extractor -hashesh from database")?; + + let all_char_vec_hashes = all_hashes + .into_iter() + .map(|hash| hash.hash().to_hex().chars().collect::<Vec<char>>()) + .collect::<Vec<Vec<_>>>(); + + // This value should be updated later, if not rust will panic in the assertion. + let mut needed_chars: usize = 1000; + 'outer: for i in 1..64 { + let i_chars: Vec<String> = all_char_vec_hashes + .iter() + .map(|vec| vec.iter().take(i).collect::<String>()) + .collect(); + + let mut uniqnes_hashmap: HashSet<String> = HashSet::new(); + for ch in i_chars { + if !uniqnes_hashmap.insert(ch) { + // The key was already in the hash map, thus we have a duplicated char and need + // at least one char more + continue 'outer; + } + } + + needed_chars = i; + break 'outer; + } + + assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); + + Ok(needed_chars) + } +} diff --git a/crates/yt/src/storage/db/get/extractor_hash.rs b/crates/yt/src/storage/db/get/extractor_hash.rs new file mode 100644 index 0000000..c8e150a --- /dev/null +++ b/crates/yt/src/storage/db/get/extractor_hash.rs @@ -0,0 +1,68 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use blake3::Hash; +use sqlx::{SqliteConnection, query}; + +use crate::{ + app::App, + storage::db::{ + extractor_hash::ExtractorHash, + video::{Video, video_from_record}, + }, +}; + +impl ExtractorHash { + pub(crate) async fn get(&self, txn: &mut SqliteConnection) -> Result<Video> { + let extractor_hash = self.hash().to_string(); + + let base = query!( + r#" + SELECT * + FROM videos + WHERE extractor_hash = ? + "#, + extractor_hash + ) + .fetch_one(txn) + .await?; + + Ok(video_from_record!(base)) + } + + pub(crate) async fn get_with_app(&self, app: &App) -> Result<Video> { + let mut txn = app.database.begin().await?; + let out = self.get(&mut txn).await?; + txn.commit().await?; + + Ok(out) + } + + pub(crate) async fn get_all(app: &App) -> Result<Vec<Self>> { + let hashes_hex = query!( + r#" + SELECT extractor_hash + FROM videos; + "# + ) + .fetch_all(&app.database) + .await?; + + Ok(hashes_hex + .iter() + .map(|hash| { + Self::from_hash(Hash::from_hex(&hash.extractor_hash).expect( + "These values started as blake3 hashes, they should stay blake3 hashes", + )) + }) + .collect()) + } +} diff --git a/crates/yt/src/storage/db/get/mod.rs b/crates/yt/src/storage/db/get/mod.rs new file mode 100644 index 0000000..4bcd066 --- /dev/null +++ b/crates/yt/src/storage/db/get/mod.rs @@ -0,0 +1,15 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod extractor_hash; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod txn_log; +pub(crate) mod video; diff --git a/crates/yt/src/storage/db/get/playlist.rs b/crates/yt/src/storage/db/get/playlist.rs new file mode 100644 index 0000000..5094523 --- /dev/null +++ b/crates/yt/src/storage/db/get/playlist.rs @@ -0,0 +1,68 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{ + playlist::{Playlist, PlaylistIndex}, + video::{Video, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +impl Playlist { + /// Get an video based in its index. + #[must_use] + pub(crate) fn get_mut(&mut self, index: PlaylistIndex) -> Option<&mut Video> { + self.videos.get_mut(Into::<usize>::into(index)) + } + + /// Create a playlist, by loading it from the database. + pub(crate) async fn create(app: &App) -> Result<Self> { + let videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + Ok(Self { videos }) + } + + /// Return the current playlist index. + /// + /// This effectively looks for the currently focused video and returns it's index. + /// + /// # Panics + /// Only if internal assertions fail. + pub(crate) fn current_index(&self) -> Option<PlaylistIndex> { + if let Some((index, _)) = self.get_focused() { + Some(index) + } else { + None + } + } + + /// Get the currently focused video, if it exists. + #[must_use] + pub(crate) fn get_focused_mut(&mut self) -> Option<(PlaylistIndex, &mut Video)> { + self.videos + .iter_mut() + .enumerate() + .find(|(_, v)| v.is_focused()) + .map(|(index, video)| (PlaylistIndex::from(index), video)) + } + + /// Get the currently focused video, if it exists. + #[must_use] + pub(crate) fn get_focused(&self) -> Option<(PlaylistIndex, &Video)> { + self.videos + .iter() + .enumerate() + .find(|(_, v)| v.is_focused()) + .map(|(index, video)| (PlaylistIndex::from(index), video)) + } +} diff --git a/crates/yt/src/storage/db/get/subscription.rs b/crates/yt/src/storage/db/get/subscription.rs new file mode 100644 index 0000000..16a6e8b --- /dev/null +++ b/crates/yt/src/storage/db/get/subscription.rs @@ -0,0 +1,49 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::collections::HashMap; + +use crate::{ + app::App, + storage::db::subscription::{Subscription, Subscriptions}, +}; + +use anyhow::Result; +use sqlx::query; +use url::Url; + +impl Subscriptions { + /// Get a list of subscriptions + pub(crate) async fn get(app: &App) -> Result<Self> { + let raw_subs = query!( + " + SELECT * + FROM subscriptions; + " + ) + .fetch_all(&app.database) + .await?; + + let subscriptions: HashMap<String, Subscription> = raw_subs + .into_iter() + .map(|sub| { + ( + sub.name.clone(), + Subscription::new( + sub.name, + Url::parse(&sub.url).expect("It was an URL, when we inserted it."), + ), + ) + }) + .collect(); + + Ok(Subscriptions(subscriptions)) + } +} diff --git a/crates/yt/src/storage/db/get/txn_log.rs b/crates/yt/src/storage/db/get/txn_log.rs new file mode 100644 index 0000000..1a6df2c --- /dev/null +++ b/crates/yt/src/storage/db/get/txn_log.rs @@ -0,0 +1,43 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{insert::Committable, txn_log::TxnLog, video::TimeStamp}, +}; + +use anyhow::Result; +use sqlx::query; + +impl<O: Committable> TxnLog<O> { + /// Get the log of all operations that have been performed. + pub(crate) async fn get(app: &App) -> Result<Self> { + let raw_ops = query!( + " + SELECT * + FROM txn_log + ORDER BY timestamp ASC; + " + ) + .fetch_all(&app.database) + .await?; + + let inner = raw_ops + .into_iter() + .filter_map(|raw_op| { + serde_json::from_str(&raw_op.operation) + .map(|parsed_op| (TimeStamp::from_secs(raw_op.timestamp), parsed_op)) + .ok() + }) + .collect(); + + Ok(TxnLog::new(inner)) + } +} diff --git a/crates/yt/src/storage/db/get/video/mod.rs b/crates/yt/src/storage/db/get/video/mod.rs new file mode 100644 index 0000000..69adb6b --- /dev/null +++ b/crates/yt/src/storage/db/get/video/mod.rs @@ -0,0 +1,261 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{fs::File, path::PathBuf}; + +use anyhow::{Context, Result, bail}; +use log::debug; +use sqlx::query; +use yt_dlp::{info_json::InfoJson, json_cast, json_try_get}; + +use crate::{ + app::App, + storage::db::video::{ + Video, VideoStatus, VideoStatusMarker, + comments::{Comments, raw::RawComment}, + video_from_record, + }, +}; + +impl Video { + /// Returns to next video which should be downloaded. This respects the priority assigned by select. + /// It does not return videos, which are already downloaded. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) async fn next_to_download(app: &App) -> Result<Option<Self>> { + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + // NOTE: The ORDER BY statement should be the same as the one in [`in_states`]. <2024-08-22> + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND cache_path IS NULL + ORDER BY priority DESC, publish_date DESC + LIMIT 1; + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record!(base))) + } + } + + /// Returns the description of the current video. + /// The returned description will be set to `<No description>` in the absence of one. + /// + /// # Errors + /// If no current video exists. + /// + /// # Panics + /// If the current video lacks the `info.json` file. + pub(crate) async fn get_current_description(app: &App) -> Result<String> { + let Some(currently_playing_video) = Video::currently_focused(app).await? else { + bail!("Could not find a currently playing video!"); + }; + + let info_json = ¤tly_playing_video.get_info_json()?.expect( + "A currently *playing* must be cached. \ + And thus the info.json should be available.", + ); + + let description = json_try_get!(info_json, "description", as_str) + .unwrap_or("<No description>") + .to_owned(); + + Ok(description) + } + + /// Returns the comments of the current video. + /// The returned [`Comments`] will be empty in the absence of comments. + /// + /// # Errors + /// If no current video exists. + /// + /// # Panics + /// If the current video lacks the `info.json` file. + pub(crate) async fn get_current_comments(app: &App) -> Result<Comments> { + let Some(currently_playing_video) = Video::currently_focused(app).await? else { + bail!("Could not find a currently playing video!"); + }; + + let info_json = ¤tly_playing_video.get_info_json()?.expect( + "A currently *playing* video must be cached. \ + And thus the info.json should be available.", + ); + + let raw_comments = if let Some(comments) = json_try_get!(info_json, "comments", as_array) { + comments + .iter() + .cloned() + .map(serde_json::from_value) + .collect::<Result<Vec<RawComment>, _>>()? + } else { + // TODO(@bpeetz): We could display a `<No-comments>` here. <2025-07-15> + + bail!( + "The video ('{}') does not have comments!", + json_try_get!(info_json, "title", as_str).unwrap_or("<No Title>") + ) + }; + + Ok(Comments::from_raw(raw_comments)) + } + + /// Optionally returns the video that is currently focused. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) async fn currently_focused(app: &App) -> Result<Option<Self>> { + let status = VideoStatusMarker::Cached.as_db_integer(); + + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND is_focused = 1 + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record!(base))) + } + } + + /// Calculate the [`info_json`] location on-disk for this video. + /// + /// Will return [`None`], if the video does not have an downloaded [`info_json`] + pub(crate) fn info_json_path(&self) -> Result<Option<PathBuf>> { + if let VideoStatus::Cached { mut cache_path, .. } = self.status.clone() { + if !cache_path.set_extension("info.json") { + bail!( + "Failed to change path extension to 'info.json': {}", + cache_path.display() + ); + } + + Ok(Some(cache_path)) + } else { + Ok(None) + } + } + + /// Fetch the [`info_json`], downloaded on-disk for this video. + /// + /// Will return [`None`], if the video does not have an downloaded [`info_json`] + pub(crate) fn get_info_json(&self) -> Result<Option<InfoJson>> { + if let Some(path) = self.info_json_path()? { + let info_json_string = File::open(path)?; + let info_json = serde_json::from_reader(&info_json_string)?; + + Ok(Some(info_json)) + } else { + Ok(None) + } + } + + /// Returns this videos `is_focused` flag if it is set. + /// + /// Will return `false` for not-downloaded videos. + pub(crate) fn is_focused(&self) -> bool { + if let VideoStatus::Cached { is_focused, .. } = &self.status { + *is_focused + } else { + false + } + } + + /// Returns the videos that are in the `allowed_states`. + /// + /// # Panics + /// Only, if assertions fail. + pub(crate) async fn in_states( + app: &App, + allowed_states: &[VideoStatusMarker], + ) -> Result<Vec<Video>> { + fn test(all_states: &[VideoStatusMarker], check: VideoStatusMarker) -> Option<i64> { + if all_states.contains(&check) { + Some(check.as_db_integer()) + } else { + None + } + } + fn states_to_string(allowed_states: &[VideoStatusMarker]) -> String { + let mut states = allowed_states + .iter() + .fold(String::from("&["), |mut acc, state| { + acc.push_str(state.as_str()); + acc.push_str(", "); + acc + }); + states = states.trim().to_owned(); + states = states.trim_end_matches(',').to_owned(); + states.push(']'); + states + } + + debug!( + "Fetching videos in the states: '{}'", + states_to_string(allowed_states) + ); + let active_pick: Option<i64> = test(allowed_states, VideoStatusMarker::Pick); + let active_watch: Option<i64> = test(allowed_states, VideoStatusMarker::Watch); + let active_cached: Option<i64> = test(allowed_states, VideoStatusMarker::Cached); + let active_watched: Option<i64> = test(allowed_states, VideoStatusMarker::Watched); + let active_drop: Option<i64> = test(allowed_states, VideoStatusMarker::Drop); + let active_dropped: Option<i64> = test(allowed_states, VideoStatusMarker::Dropped); + + // NOTE: The ORDER BY statement should be the same as the one in [`next_to_download`]. <2024-08-22> + let videos = query!( + r" + SELECT * + FROM videos + WHERE status IN (?,?,?,?,?,?) + ORDER BY priority DESC, publish_date DESC; + ", + active_pick, + active_watch, + active_cached, + active_watched, + active_drop, + active_dropped, + ) + .fetch_all(&app.database) + .await + .with_context(|| { + format!( + "Failed to query videos with states: '{}'", + states_to_string(allowed_states) + ) + })?; + + let real_videos: Vec<Video> = videos + .iter() + .map(|base| -> Video { video_from_record!(base) }) + .collect(); + + Ok(real_videos) + } +} diff --git a/crates/yt/src/storage/db/insert/maintenance.rs b/crates/yt/src/storage/db/insert/maintenance.rs new file mode 100644 index 0000000..d87c1ae --- /dev/null +++ b/crates/yt/src/storage/db/insert/maintenance.rs @@ -0,0 +1,38 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + storage::db::{ + insert::Operations, + video::{Video, VideoStatus, VideoStatusMarker}, + }, +}; + +use anyhow::Result; + +/// Remove the downloaded paths from videos in the db, that no longer exist on the file system. +pub(crate) async fn clear_stale_downloaded_paths(app: &App) -> Result<()> { + let mut cached_videos = Video::in_states(app, &[VideoStatusMarker::Cached]).await?; + + let mut ops = Operations::new("DbMaintain: init"); + for vid in &mut cached_videos { + if let VideoStatus::Cached { cache_path, .. } = &vid.status { + if !cache_path.exists() { + vid.remove_download_path(&mut ops); + } + } else { + unreachable!("We only asked for cached videos.") + } + } + ops.commit(app).await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/db/insert/mod.rs b/crates/yt/src/storage/db/insert/mod.rs new file mode 100644 index 0000000..3458608 --- /dev/null +++ b/crates/yt/src/storage/db/insert/mod.rs @@ -0,0 +1,115 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::mem; + +use crate::app::App; + +use anyhow::Result; +use chrono::Utc; +use log::{debug, trace}; +use serde::{Serialize, de::DeserializeOwned}; +use sqlx::{SqliteConnection, query}; + +pub(crate) mod maintenance; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod video; + +pub(crate) trait Committable: + Sized + std::fmt::Debug + Serialize + DeserializeOwned +{ + async fn commit(self, txn: &mut SqliteConnection) -> Result<()>; +} + +#[derive(Debug)] +pub(crate) struct Operations<O: Committable> { + name: &'static str, + ops: Vec<O>, +} + +impl<O: Committable> Default for Operations<O> { + fn default() -> Self { + Self::new("<default impl>") + } +} + +impl<O: Committable> Operations<O> { + #[must_use] + pub(crate) fn new(name: &'static str) -> Self { + Self { + name, + ops: Vec::new(), + } + } + + pub(crate) async fn commit(mut self, app: &App) -> Result<()> { + let ops = mem::take(&mut self.ops); + + if ops.is_empty() { + return Ok(()); + } + + trace!("Begin commit of {}", self.name); + let mut txn = app.database.begin().await?; + + for op in ops { + trace!("Commiting operation: {op:?}"); + add_operation_to_txn_log(&op, &mut txn).await?; + op.commit(&mut txn).await?; + } + + txn.commit().await?; + trace!("End commit of {}", self.name); + + Ok(()) + } + + pub(crate) fn push(&mut self, op: O) { + self.ops.push(op); + } +} + +impl<O: Committable> Drop for Operations<O> { + fn drop(&mut self) { + assert!( + self.ops.is_empty(), + "Trying to drop uncommitted operations (name: {}) ({:#?}). This is a bug.", + self.name, + self.ops + ); + } +} + +async fn add_operation_to_txn_log<O: Committable>( + operation: &O, + txn: &mut SqliteConnection, +) -> Result<()> { + debug!("Adding operation to txn log: {operation:?}"); + + let now = Utc::now().timestamp(); + let operation = serde_json::to_string(&operation).expect("should be serializable"); + + query!( + r#" + INSERT INTO txn_log ( + timestamp, + operation + ) + VALUES (?, ?); + "#, + now, + operation, + ) + .execute(txn) + .await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/db/insert/playlist.rs b/crates/yt/src/storage/db/insert/playlist.rs new file mode 100644 index 0000000..4d3e140 --- /dev/null +++ b/crates/yt/src/storage/db/insert/playlist.rs @@ -0,0 +1,222 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{cmp::Ordering, time::Duration}; + +use anyhow::{Context, Result}; +use colors::Colorize; +use libmpv2::Mpv; +use log::{debug, trace}; + +use crate::{ + app::App, + storage::db::{ + insert::{Operations, video::Operation}, + playlist::{Playlist, PlaylistIndex}, + video::VideoStatus, + }, +}; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum VideoTransition { + Watched, + Picked, +} + +impl Playlist { + pub(crate) fn mark_current_done( + &mut self, + app: &App, + mpv: &Mpv, + new_state: VideoTransition, + ops: &mut Operations<Operation>, + ) -> Result<()> { + let (current_index, current_video) = self + .get_focused_mut() + .expect("This should be some at this point"); + + debug!( + "Playlist handler will mark video '{}' {:?}.", + current_video.title, new_state + ); + + match new_state { + VideoTransition::Watched => current_video.set_watched(ops), + VideoTransition::Picked => current_video.set_status(VideoStatus::Pick, ops), + } + + self.save_watch_progress(mpv, current_index, ops); + + self.videos.remove(Into::<usize>::into(current_index)); + + { + // Decide which video to mark focused now. + let index = usize::from(current_index); + let playlist_length = self.len(); + + if playlist_length == 0 { + // There are no new videos to mark focused. + } else { + let index = match index.cmp(&playlist_length) { + Ordering::Greater => { + unreachable!( + "The index '{index}' cannot exceed the \ + playlist length '{playlist_length}' as indices are 0 based." + ); + } + Ordering::Less => { + // The index is still valid. + // Therefore, we keep the user at this position. + index + } + Ordering::Equal => { + // The index is pointing to the end of the playlist. We could either go the second + // to last entry (i.e., one entry back) or wrap around to the start. + // We wrap around. + 0 + } + }; + + let next = self + .get_mut(PlaylistIndex::from(index)) + .expect("We checked that the index is still good"); + next.set_focused(true, ops); + } + + // Tell mpv about our decision. + self.resync_with_mpv(app, mpv)?; + } + + Ok(()) + } + + /// Sync the mpv playlist with this playlist. + pub(crate) fn resync_with_mpv(&self, app: &App, mpv: &Mpv) -> Result<()> { + fn get_playlist_count(mpv: &Mpv) -> Result<usize> { + mpv.get_property::<i64>("playlist/count") + .context("Failed to get mpv playlist len") + .map(|count| { + usize::try_from(count).expect("The playlist_count should always be positive") + }) + } + + if get_playlist_count(mpv)? != 0 { + // We could also use `loadlist`, but that would require use to start a unix socket or even + // write all the video paths to a file beforehand + mpv.command("playlist-clear", &[])?; + mpv.command("playlist-remove", &["current"])?; + } + + assert_eq!( + get_playlist_count(mpv)?, + 0, + "The playlist should be empty at this point." + ); + + debug!("MpvReload: Adding {} videos to playlist.", self.len()); + + self.videos + .iter() + .enumerate() + .try_for_each(|(index, video)| { + let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + else { + unreachable!("All of the videos in a playlist are cached"); + }; + + let options = format!( + "speed={},start={}", + video + .playback_speed + .unwrap_or(app.config.select.playback_speed), + i64::try_from(video.watch_progress.as_secs()) + .expect("This should not overflow"), + ); + + mpv.command( + "loadfile", + &[ + cache_path.to_str().with_context(|| { + format!( + "Failed to parse the video cache path ('{}') as valid utf8", + cache_path.display() + ) + })?, + "append-play", + "-1", // Not used for `append-play`, but needed for the next args to take effect. + options.as_str(), + ], + )?; + + if *is_focused { + debug!("MpvReload: Setting playlist position to {index}"); + mpv.set_property("playlist-pos", index.to_string().as_str())?; + } + + Ok::<(), anyhow::Error>(()) + })?; + + Ok(()) + } + + pub(crate) fn save_current_watch_progress( + &mut self, + mpv: &Mpv, + ops: &mut Operations<Operation>, + ) { + let (index, _) = self + .get_focused_mut() + .expect("This should be some at this point"); + + self.save_watch_progress(mpv, index, ops); + } + + /// Saves the `watch_progress` of a video at the index. + pub(crate) fn save_watch_progress( + &mut self, + mpv: &Mpv, + at: PlaylistIndex, + ops: &mut Operations<Operation>, + ) { + let current_video = self + .get_mut(at) + .expect("We should never produce invalid playlist indices"); + + let watch_progress = match mpv.get_property::<i64>("time-pos") { + Ok(time) => u64::try_from(time) + .expect("This conversion should never fail as the `time-pos` property is positive"), + Err(err) => { + // We cannot hard error here, as we would open us to an race condition between mpv + // changing the current video and we saving it. + trace!( + "While trying to save the watch progress for the current video: \ + Failed to get the watchprogress of the currently playling video: \ + (This is probably expected, nevertheless showing the raw error) \ + {err}" + ); + + return; + } + }; + + let watch_progress = Duration::from_secs(watch_progress); + + debug!( + "Setting the watch progress for the current_video '{}' to {}s", + current_video.title_fmt().render(false), + watch_progress.as_secs(), + ); + + current_video.set_watch_progress(watch_progress, ops); + } +} diff --git a/crates/yt/src/storage/db/insert/subscription.rs b/crates/yt/src/storage/db/insert/subscription.rs new file mode 100644 index 0000000..d25a209 --- /dev/null +++ b/crates/yt/src/storage/db/insert/subscription.rs @@ -0,0 +1,95 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::storage::db::{ + insert::{Committable, Operations}, + subscription::{Subscription, Subscriptions}, +}; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use sqlx::query; + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum Operation { + Add(Subscription), + Remove(Subscription), +} + +impl Committable for Operation { + async fn commit(self, txn: &mut sqlx::SqliteConnection) -> Result<()> { + match self { + Operation::Add(subscription) => { + let url = subscription.url.as_str(); + + query!( + " + INSERT INTO subscriptions ( + name, + url + ) VALUES (?, ?); + ", + subscription.name, + url + ) + .execute(txn) + .await?; + + println!( + "Subscribed to '{}' at '{}'", + subscription.name, subscription.url + ); + Ok(()) + } + Operation::Remove(subscription) => { + let output = query!( + " + DELETE FROM subscriptions + WHERE name = ? + ", + subscription.name, + ) + .execute(txn) + .await?; + + assert_eq!( + output.rows_affected(), + 1, + "The removed subscription query did effect more (or less) than one row. This is a bug." + ); + + println!( + "Unsubscribed from '{}' at '{}'", + subscription.name, subscription.url + ); + + Ok(()) + } + } + } +} + +impl Subscription { + pub(crate) fn add(self, ops: &mut Operations<Operation>) { + ops.push(Operation::Add(self)); + } + + pub(crate) fn remove(self, ops: &mut Operations<Operation>) { + ops.push(Operation::Remove(self)); + } +} + +impl Subscriptions { + pub(crate) fn remove(self, ops: &mut Operations<Operation>) { + for sub in self.0.into_values() { + ops.push(Operation::Remove(sub)); + } + } +} diff --git a/crates/yt/src/storage/db/insert/video/mod.rs b/crates/yt/src/storage/db/insert/video/mod.rs new file mode 100644 index 0000000..da62e37 --- /dev/null +++ b/crates/yt/src/storage/db/insert/video/mod.rs @@ -0,0 +1,610 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + path::{Path, PathBuf}, + time, +}; + +use crate::storage::db::{ + extractor_hash::ExtractorHash, + insert::{Committable, Operations}, + video::{Priority, Video, VideoStatus, VideoStatusMarker}, +}; + +use anyhow::{Context, Result}; +use chrono::Utc; +use log::debug; +use serde::{Deserialize, Serialize}; +use sqlx::query; +use tokio::fs; + +use super::super::video::TimeStamp; + +const fn is_focused_to_value(is_focused: bool) -> Option<i8> { + if is_focused { Some(1) } else { None } +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum Operation { + Add { + description: Option<String>, + title: String, + parent_subscription_name: Option<String>, + thumbnail_url: Option<String>, + url: String, + extractor_hash: String, + status: i64, + cache_path: Option<String>, + is_focused: Option<i8>, + duration: Option<f64>, + last_status_change: i64, + publish_date: Option<i64>, + watch_progress: i64, + }, + // TODO(@bpeetz): Could both the {`Set`,`Remove`}`DownloadPath` ops, be merged into SetStatus + // {`Cached`,`Watch`}? <2025-07-14> + SetDownloadPath { + video: ExtractorHash, + path: PathBuf, + }, + RemoveDownloadPath { + video: ExtractorHash, + }, + SetStatus { + video: ExtractorHash, + status: VideoStatus, + }, + SetPriority { + video: ExtractorHash, + priority: Priority, + }, + SetPlaybackSpeed { + video: ExtractorHash, + playback_speed: f64, + }, + SetSubtitleLangs { + video: ExtractorHash, + subtitle_langs: String, + }, + SetWatchProgress { + video: ExtractorHash, + watch_progress: time::Duration, + }, + SetIsFocused { + video: ExtractorHash, + is_focused: bool, + }, +} + +impl Committable for Operation { + #[allow(clippy::too_many_lines)] + async fn commit(self, txn: &mut sqlx::SqliteConnection) -> Result<()> { + match self { + Operation::SetDownloadPath { video, path } => { + debug!("Setting cache path from '{video}' to '{}'", path.display()); + + let path_str = path.display().to_string(); + let extractor_hash = video.hash().to_string(); + let status = VideoStatusMarker::Cached.as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = ?, status = ? + WHERE extractor_hash = ?; + "#, + path_str, + status, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::RemoveDownloadPath { video } => { + let extractor_hash = video.hash().to_string(); + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + let old = video.get(&mut *txn).await?; + + debug!("Deleting download path of '{video}' ({}).", old.title); + + if let VideoStatus::Cached { cache_path, .. } = &old.status { + if let Ok(true) = cache_path.try_exists() { + fs::remove_file(cache_path).await?; + } + + { + let info_json_path = old.info_json_path()?.expect("Is downloaded"); + + if let Ok(true) = info_json_path.try_exists() { + fs::remove_file(info_json_path).await?; + } + } + + { + if old.subtitle_langs.is_some() { + // TODO(@bpeetz): Also clean-up the downloaded subtitle files. <2025-07-05> + } + } + } else { + unreachable!( + "A video cannot have a download path deletion \ + queued without being marked as Cached." + ); + } + + query!( + r#" + UPDATE videos + SET cache_path = NULL, status = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + status, + None::<i32>, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetStatus { video, status } => { + let extractor_hash = video.hash().to_string(); + + let old = video.get(&mut *txn).await?; + + let old_marker = old.status.as_marker(); + + let (cache_path, is_focused) = { + fn cache_path_to_string(path: &Path) -> Result<String> { + Ok(path + .to_str() + .with_context(|| { + format!( + "Failed to parse cache path ('{}') as utf8 string", + path.display() + ) + })? + .to_owned()) + } + + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &status + { + ( + Some(cache_path_to_string(cache_path)?), + is_focused_to_value(*is_focused), + ) + } else { + (None, None) + } + }; + + let new_status = status.as_marker(); + + assert_ne!( + old_marker, new_status, + "We should have never generated this operation" + ); + + let now = Utc::now().timestamp(); + + debug!( + "Changing status of video ('{}' {extractor_hash}) \ + from {old_marker:#?} to {new_status:#?}.", + old.title + ); + + let new_status = new_status.as_db_integer(); + query!( + r#" + UPDATE videos + SET status = ?, last_status_change = ?, cache_path = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + new_status, + now, + cache_path, + is_focused, + extractor_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetPriority { video, priority } => { + let extractor_hash = video.hash().to_string(); + + let new_priority = priority.as_db_integer(); + query!( + r#" + UPDATE videos + SET priority = ? + WHERE extractor_hash = ?; + "#, + new_priority, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetWatchProgress { + video, + watch_progress, + } => { + let video_extractor_hash = video.hash().to_string(); + let watch_progress = i64::try_from(watch_progress.as_secs()) + .expect("This should never exceed its bounds"); + + query!( + r#" + UPDATE videos + SET watch_progress = ? + WHERE extractor_hash = ?; + "#, + watch_progress, + video_extractor_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetPlaybackSpeed { + video, + playback_speed, + } => { + let extractor_hash = video.hash().to_string(); + + query!( + r#" + UPDATE videos + SET playback_speed = ? + WHERE extractor_hash = ?; + "#, + playback_speed, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetSubtitleLangs { + video, + subtitle_langs, + } => { + let extractor_hash = video.hash().to_string(); + + query!( + r#" + UPDATE videos + SET subtitle_langs = ? + WHERE extractor_hash = ?; + "#, + subtitle_langs, + extractor_hash + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::SetIsFocused { video, is_focused } => { + debug!("Set is_focused of video: '{video}' to {is_focused}"); + let new_hash = video.hash().to_string(); + let new_is_focused = is_focused_to_value(is_focused); + + query!( + r#" + UPDATE videos + SET is_focused = ? + WHERE extractor_hash = ?; + "#, + new_is_focused, + new_hash, + ) + .execute(txn) + .await?; + + Ok(()) + } + Operation::Add { + parent_subscription_name, + thumbnail_url, + url, + extractor_hash, + status, + cache_path, + is_focused, + duration, + last_status_change, + publish_date, + watch_progress, + description, + title, + } => { + query!( + r#" + INSERT INTO videos ( + description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + title, + url, + watch_progress, + cache_path + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + "#, + description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + title, + url, + watch_progress, + cache_path, + ) + .execute(txn) + .await?; + + Ok(()) + } + } + } +} + +impl Video { + /// Add this in-memory video to the db. + pub(crate) fn add(self, ops: &mut Operations<Operation>) -> Result<Self> { + let description = self.description.clone(); + let title = self.title.clone(); + let parent_subscription_name = self.parent_subscription_name.clone(); + + let thumbnail_url = self.thumbnail_url.as_ref().map(ToString::to_string); + + let url = self.url.to_string(); + let extractor_hash = self.extractor_hash.hash().to_string(); + + let status = self.status.as_marker().as_db_integer(); + let (cache_path, is_focused) = if let VideoStatus::Cached { + cache_path, + is_focused, + } = &self.status + { + ( + Some( + cache_path + .to_str() + .with_context(|| { + format!( + "Failed to prase cache path '{}' as utf-8 string", + cache_path.display() + ) + })? + .to_string(), + ), + is_focused_to_value(*is_focused), + ) + } else { + (None, None) + }; + + let duration: Option<f64> = self.duration.as_secs_f64(); + let last_status_change: i64 = self.last_status_change.as_secs(); + let publish_date: Option<i64> = self.publish_date.map(TimeStamp::as_secs); + let watch_progress: i64 = + i64::try_from(self.watch_progress.as_secs()).expect("This should never exceed a u32"); + + ops.push(Operation::Add { + description, + title, + parent_subscription_name, + thumbnail_url, + url, + extractor_hash, + status, + cache_path, + is_focused, + duration, + last_status_change, + publish_date, + watch_progress, + }); + + Ok(self) + } + + /// Set the download path of a video. + /// + /// # Note + /// This will also set the status to `Cached`. + pub(crate) fn set_download_path(&mut self, path: &Path, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { cache_path, .. } = &mut self.status { + if cache_path != path { + // Update the in-memory video. + path.clone_into(cache_path); + + ops.push(Operation::SetDownloadPath { + video: self.extractor_hash, + path: path.to_owned(), + }); + } + } else { + self.status = VideoStatus::Cached { + cache_path: path.to_owned(), + is_focused: false, + }; + + ops.push(Operation::SetDownloadPath { + video: self.extractor_hash, + path: path.to_owned(), + }); + } + } + + /// Remove the download path of a video. + /// + /// # Note + /// This will also set the status to `Watch`. + /// + /// # Panics + /// If the status is not `Cached`. + pub(crate) fn remove_download_path(&mut self, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { .. } = &mut self.status { + self.status = VideoStatus::Watch; + ops.push(Operation::RemoveDownloadPath { + video: self.extractor_hash, + }); + } else { + unreachable!("Can only remove the path from a `Cached` video"); + } + } + + /// Update the `is_focused` flag of this video. + /// + /// # Note + /// It will only actually add operations, if the `is_focused` flag is different. + /// + /// # Panics + /// If the status is not `Cached`. + pub(crate) fn set_focused(&mut self, new_is_focused: bool, ops: &mut Operations<Operation>) { + if let VideoStatus::Cached { is_focused, .. } = &mut self.status { + if *is_focused != new_is_focused { + *is_focused = new_is_focused; + + ops.push(Operation::SetIsFocused { + video: self.extractor_hash, + is_focused: new_is_focused, + }); + } + } else { + unreachable!("Can only change `is_focused` on a Cached video."); + } + } + + /// Set the status of this video. + /// + /// # Note + /// This will not actually add any operations, if the new status equals the old one. + pub(crate) fn set_status(&mut self, status: VideoStatus, ops: &mut Operations<Operation>) { + if self.status != status { + status.clone_into(&mut self.status); + + ops.push(Operation::SetStatus { + video: self.extractor_hash, + status, + }); + } + } + + /// Set the priority of this video. + /// + /// # Note + /// This will not actually add any operations, if the new priority equals the old one. + pub(crate) fn set_priority(&mut self, priority: Priority, ops: &mut Operations<Operation>) { + if self.priority != priority { + self.priority = priority; + + ops.push(Operation::SetPriority { + video: self.extractor_hash, + priority, + }); + } + } + + /// Set the watch progress. + /// + /// # Note + /// This will not actually add any operations, + /// if the new watch progress equals the old one. + pub(crate) fn set_watch_progress( + &mut self, + watch_progress: time::Duration, + ops: &mut Operations<Operation>, + ) { + if self.watch_progress != watch_progress { + self.watch_progress = watch_progress; + + ops.push(Operation::SetWatchProgress { + video: self.extractor_hash, + watch_progress, + }); + } + } + + /// Set the playback speed of this video. + /// + /// # Note + /// This will not actually add any operations, if the new speed equals the old one. + pub(crate) fn set_playback_speed( + &mut self, + playback_speed: f64, + ops: &mut Operations<Operation>, + ) { + if self.playback_speed != Some(playback_speed) { + self.playback_speed = Some(playback_speed); + + ops.push(Operation::SetPlaybackSpeed { + video: self.extractor_hash, + playback_speed, + }); + } + } + + /// Set the subtitle langs of this video. + /// + /// # Note + /// This will not actually add any operations, if the new langs equal the old one. + pub(crate) fn set_subtitle_langs( + &mut self, + subtitle_langs: String, + ops: &mut Operations<Operation>, + ) { + if self.subtitle_langs.as_ref() != Some(&subtitle_langs) { + self.subtitle_langs = Some(subtitle_langs.clone()); + + ops.push(Operation::SetSubtitleLangs { + video: self.extractor_hash, + subtitle_langs, + }); + } + } + + /// Mark this video watched. + /// This will both set the status to `Watched` and the `cache_path` to Null. + /// + /// # Panics + /// Only if assertions fail. + pub(crate) fn set_watched(&mut self, ops: &mut Operations<Operation>) { + self.remove_download_path(ops); + self.set_status(VideoStatus::Watched, ops); + } +} diff --git a/crates/yt/src/storage/db/mod.rs b/crates/yt/src/storage/db/mod.rs new file mode 100644 index 0000000..926bab0 --- /dev/null +++ b/crates/yt/src/storage/db/mod.rs @@ -0,0 +1,18 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub(crate) mod get; +pub(crate) mod insert; + +pub(crate) mod extractor_hash; +pub(crate) mod playlist; +pub(crate) mod subscription; +pub(crate) mod txn_log; +pub(crate) mod video; diff --git a/crates/yt/src/storage/db/playlist/mod.rs b/crates/yt/src/storage/db/playlist/mod.rs new file mode 100644 index 0000000..7366e8e --- /dev/null +++ b/crates/yt/src/storage/db/playlist/mod.rs @@ -0,0 +1,59 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::ops::Add; + +use crate::storage::db::video::Video; + +/// Zero-based index into the internal playlist. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PlaylistIndex(usize); + +impl From<PlaylistIndex> for usize { + fn from(value: PlaylistIndex) -> Self { + value.0 + } +} + +impl From<usize> for PlaylistIndex { + fn from(value: usize) -> Self { + Self(value) + } +} + +impl Add<usize> for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl Add for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +/// A representation of the internal Playlist +#[derive(Debug)] +pub(crate) struct Playlist { + pub(crate) videos: Vec<Video>, +} + +impl Playlist { + /// Returns the number of videos in the playlist + #[must_use] + pub(crate) fn len(&self) -> usize { + self.videos.len() + } +} diff --git a/crates/yt/src/storage/db/subscription.rs b/crates/yt/src/storage/db/subscription.rs new file mode 100644 index 0000000..c111b52 --- /dev/null +++ b/crates/yt/src/storage/db/subscription.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::collections::HashMap; + +use anyhow::Result; +use log::debug; +use serde::{Deserialize, Serialize}; +use url::Url; +use yt_dlp::{json_cast, json_try_get, options::YoutubeDLOptions}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct Subscription { + /// The human readable name of this subscription + pub(crate) name: String, + + /// The URL this subscription subscribes to + pub(crate) url: Url, +} + +impl Subscription { + #[must_use] + pub(crate) fn new(name: String, url: Url) -> Self { + Self { name, url } + } +} + +#[derive(Default, Debug)] +pub(crate) struct Subscriptions(pub(crate) HashMap<String, Subscription>); + +/// Check whether an URL could be used as a subscription URL +pub(crate) async fn check_url(url: Url) -> Result<bool> { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + debug!("{info:#?}"); + + Ok(json_try_get!(info, "_type", as_str) == Some("playlist")) +} diff --git a/crates/yt/src/storage/db/txn_log.rs b/crates/yt/src/storage/db/txn_log.rs new file mode 100644 index 0000000..64884b0 --- /dev/null +++ b/crates/yt/src/storage/db/txn_log.rs @@ -0,0 +1,24 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::storage::db::{insert::Committable, video::TimeStamp}; + +pub(crate) struct TxnLog<O: Committable> { + inner: Vec<(TimeStamp, O)>, +} + +impl<O: Committable> TxnLog<O> { + pub(crate) fn new(inner: Vec<(TimeStamp, O)>) -> Self { + Self { inner } + } + pub(crate) fn inner(&self) -> &[(TimeStamp, O)] { + &self.inner + } +} diff --git a/crates/yt/src/storage/db/video/comments/display.rs b/crates/yt/src/storage/db/video/comments/display.rs new file mode 100644 index 0000000..c372603 --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/display.rs @@ -0,0 +1,127 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::Write; + +use chrono::{Local, TimeZone}; +use chrono_humanize::{Accuracy, HumanTime, Tense}; +use colors::{Colorize, IntoCanvas}; + +use crate::{ + output::format_text, + storage::db::video::comments::{Comment, Comments}, +}; + +impl Comments { + pub(crate) fn render(&self, use_color: bool) -> String { + self.render_help(use_color) + .expect("This should never fail.") + } + + fn render_help(&self, use_color: bool) -> Result<String, std::fmt::Error> { + fn format( + comment: &Comment, + f: &mut String, + ident_count: u32, + color: bool, + ) -> std::fmt::Result { + let ident = &(0..ident_count).map(|_| " ").collect::<String>(); + let value = &comment.value; + + f.write_str(ident)?; + + write!( + f, + "{}", + if value.author_is_uploader { + (&value.author).bold().bright_red().render(color) + } else { + (&value.author).purple().render(color) + } + )?; + + if value.edited || value.is_favorited { + f.write_str("[")?; + if value.edited { + f.write_str("")?; + } + if value.edited && value.is_favorited { + f.write_str(" ")?; + } + if value.is_favorited { + f.write_str("")?; + } + f.write_str("]")?; + } + + write!( + f, + " {}", + HumanTime::from( + Local + .timestamp_opt(value.timestamp, 0) + .single() + .expect("This should be valid") + ) + .to_text_en(Accuracy::Rough, Tense::Past) + .bold() + .cyan() + .render(color) + )?; + + write!( + f, + " [{}]", + comment.value.like_count.bold().red().render(color) + )?; + + f.write_str(":\n")?; + f.write_str(ident)?; + + f.write_str( + &format_text( + value.text.trim(), + Some( + termsize::get().map_or(90, |ts| ts.cols) + - u16::try_from(ident_count).expect("Should never overflow"), + ), + ) + .trim() + .replace('\n', &format!("\n{ident}")), + )?; + f.write_str("\n")?; + + if comment.replies.is_empty() { + f.write_str("\n")?; + } else { + let mut children = comment.replies.clone(); + children.sort_by(|a, b| a.value.timestamp.cmp(&b.value.timestamp)); + + for child in children { + format(&child, f, ident_count + 4, color)?; + } + } + + Ok(()) + } + + let mut f = String::new(); + + if !&self.inner.is_empty() { + let mut children = self.inner.clone(); + children.sort_by(|a, b| b.value.like_count.cmp(&a.value.like_count)); + + for child in children { + format(&child, &mut f, 0, use_color)?; + } + } + Ok(f) + } +} diff --git a/crates/yt/src/storage/db/video/comments/mod.rs b/crates/yt/src/storage/db/video/comments/mod.rs new file mode 100644 index 0000000..41a03be --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/mod.rs @@ -0,0 +1,202 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::mem; + +use regex::{Captures, Regex}; + +use crate::storage::db::video::comments::raw::{Parent, RawComment}; + +pub(crate) mod display; +pub(crate) mod raw; + +#[cfg(test)] +mod tests; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Comment { + value: RawComment, + replies: Vec<Self>, +} + +#[derive(Debug, Default, PartialEq)] +pub(crate) struct Comments { + inner: Vec<Comment>, +} + +impl Comments { + pub(crate) fn from_raw(raw: Vec<RawComment>) -> Self { + let mut me = Self::default(); + + // Apply the parent -> child mapping yt provides us with. + for raw_comment in raw { + if let Parent::Id(id) = &raw_comment.parent { + me.insert(&(id.clone()), Comment::from(raw_comment)); + } else { + me.inner.push(Comment::from(raw_comment)); + } + } + + { + // Sort the final comments chronologically. + // This ensures that replies are matched with the comment they actually replied to and + // not a later comment from the same author. + for comment in &mut me.inner { + comment + .replies + .sort_by_key(|comment| comment.value.timestamp); + + for reply in &comment.replies { + assert!(reply.replies.is_empty()); + } + } + } + + { + let find_reply_indicator = + Regex::new(r"\u{200b}?(@[^\t\s]+)\u{200b}?").expect("This is hardcoded"); + + // Try to re-construct the replies for the reply comments. + for comment in &mut me.inner { + let previous_replies = mem::take(&mut comment.replies); + + let mut reply_tree = Comments::default(); + + for reply in previous_replies { + // We try to reconstruct the parent child relation ship by looking (naively) + // for a reply indicator. Currently, this is just the `@<some_name>`, as yt + // seems to insert that by default if you press `reply-to` in their clients. + // + // This follows these steps: + // - Does this reply have a “reply indicator”? + // - If yes, try to resolve the indicator. + // - If it is resolvable, add this reply to the [`Comment`] it resolved to. + // - If not, keep the comment as reply. + + if let Some(reply_indicator_matches) = + find_reply_indicator.captures(&reply.value.text.clone()) + { + // We found a reply indicator. + // First we traverse the current `reply_tree` in reversed order to find a + // match, than we check if the reply indicator matches the reply tree root + // and afterward we declare it unmatching and add it as toplevel. + + let reply_target_author = reply_indicator_matches + .get(1) + .expect("This should also exist") + .as_str(); + + if let Some(parent) = reply_tree.find_author_mut(reply_target_author) { + parent + .replies + .push(comment_from_reply(reply, &reply_indicator_matches)); + } else if comment.value.author == reply_target_author { + reply_tree + .add_toplevel(comment_from_reply(reply, &reply_indicator_matches)); + } else { + eprintln!( + "Failed to find a parent for ('{}') both directly \ + and via replies! The reply text was:\n'{}'\n", + reply_target_author, reply.value.text + ); + reply_tree.add_toplevel(reply); + } + } else { + // The comment text did not contain a reply indicator, so add it as + // toplevel. + reply_tree.add_toplevel(reply); + } + } + + comment.replies = reply_tree.inner; + } + } + + me + } + + fn add_toplevel(&mut self, value: Comment) { + self.inner.push(value); + } + + fn insert(&mut self, id: &str, value: Comment) { + let parent = self + .inner + .iter_mut() + .find(|c| c.value.id.id == id) + .expect("One of these should exist"); + + parent.replies.push(value); + } + + fn find_author_mut(&mut self, reply_target_author: &str) -> Option<&mut Comment> { + fn perform_check<'a>( + comment: &'a mut Comment, + reply_target_author: &str, + ) -> Option<&'a mut Comment> { + // TODO(@bpeetz): This is a workaround until rust has lexiographic lifetime support. <2025-07-18> + fn find_in_replies<'a>( + comment: &'a mut Comment, + reply_target_author: &str, + ) -> Option<&'a mut Comment> { + comment + .replies + .iter_mut() + .rev() + .find_map(|reply: &mut Comment| perform_check(reply, reply_target_author)) + } + let comment_author_matches_target = comment.value.author == reply_target_author; + + match find_in_replies(comment, reply_target_author) { + Some(_) => Some( + // PERFORMANCE(@bpeetz): We should not need to run this code twice. <2025-07-18> + find_in_replies(comment, reply_target_author) + .expect("We already had a Some result for this."), + ), + None if comment_author_matches_target => Some(comment), + None => None, + } + } + + for comment in self.inner.iter_mut().rev() { + if let Some(output) = perform_check(comment, reply_target_author) { + return Some(output); + } + } + + None + } +} +fn comment_from_reply(reply: Comment, reply_indicator_matches: &Captures<'_>) -> Comment { + Comment::from(RawComment { + text: { + // Remove the `@<some_name>` for the comment text. + let full_match = reply_indicator_matches + .get(0) + .expect("This will always exist"); + + let text = reply.value.text[0..full_match.start()].to_owned() + + &reply.value.text[full_match.end()..]; + + text.trim_matches(|c: char| c == '\u{200b}' || c == '\u{2060}' || c.is_whitespace()) + .to_owned() + }, + ..reply.value + }) +} + +impl From<RawComment> for Comment { + fn from(value: RawComment) -> Self { + Self { + value, + replies: vec![], + } + } +} diff --git a/crates/yt/src/storage/db/video/comments/raw.rs b/crates/yt/src/storage/db/video/comments/raw.rs new file mode 100644 index 0000000..3b7f40f --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/raw.rs @@ -0,0 +1,87 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use serde::{Deserialize, Deserializer}; +use url::Url; + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub(crate) struct Id { + pub(crate) id: String, +} +impl From<String> for Id { + fn from(value: String) -> Self { + Self { + // Take the last element if the string is split with dots, otherwise take the full id + id: value.split('.').next_back().unwrap_or(&value).to_owned(), + } + } +} + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub(crate) enum Parent { + Root, + Id(String), +} + +impl From<String> for Parent { + fn from(value: String) -> Self { + if value == "root" { + Self::Root + } else { + Self::Id(value) + } + } +} + +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[allow(clippy::struct_excessive_bools)] +pub(crate) struct RawComment { + pub(crate) id: Id, + pub(crate) text: String, + #[serde(default = "zero")] + pub(crate) like_count: u32, + pub(crate) is_pinned: bool, + pub(crate) author_id: String, + #[serde(default = "unknown")] + pub(crate) author: String, + pub(crate) author_is_verified: bool, + pub(crate) author_thumbnail: Url, + pub(crate) parent: Parent, + #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] + pub(crate) edited: bool, + // Can't also be deserialized, as it's already used in 'edited' + // _time_text: String, + pub(crate) timestamp: i64, + pub(crate) author_url: Option<Url>, + pub(crate) author_is_uploader: bool, + pub(crate) is_favorited: bool, +} + +fn unknown() -> String { + "<Unknown>".to_string() +} +fn zero() -> u32 { + 0 +} +fn edited_from_time_text<'de, D>(d: D) -> Result<bool, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(d)?; + if s.contains(" (edited)") { + Ok(true) + } else { + Ok(false) + } +} diff --git a/crates/yt/src/storage/db/video/comments/tests.rs b/crates/yt/src/storage/db/video/comments/tests.rs new file mode 100644 index 0000000..03e3597 --- /dev/null +++ b/crates/yt/src/storage/db/video/comments/tests.rs @@ -0,0 +1,249 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use pretty_assertions::assert_eq; +use url::Url; + +use crate::storage::db::video::comments::{ + Comment, Comments, RawComment, + raw::{Id, Parent}, +}; + +/// Generate both an [`expected`] and an [`input`] value from an expected comment expression. +macro_rules! mk_comments { + () => {{ + let input: Vec<RawComment> = vec![]; + let expected: Comments = Comments { + inner: vec![], + }; + + (input, expected) + }}; + + ( + $( + parent: $parent:expr, $actual_parent:ident, + ( + @ $name:ident : $comment:literal + $( + $reply_chain:tt + )* + ) + )+ + ) => {{ + let (nested_input, _) = mk_comments!( + $( + $( + parent: $parent, $name, + $reply_chain + )* + )+ + ); + + let mut input: Vec<RawComment> = vec![ + $( + mk_comments!(@to_raw input $name $comment $parent, $actual_parent) + ),+ + ]; + input.extend(nested_input); + + let expected: Comments = Comments { + inner: vec![ + $( + Comment { + value: mk_comments!(@to_raw expected $name $comment $parent, $actual_parent), + replies: { + let (_, nested_expected) = mk_comments!( + $( + parent: $parent, $name, + $reply_chain + )* + ); + + nested_expected.inner + }, + } + ),+ + ] + }; + + (input, expected) + }}; + ( + $( + ( + @ $name:ident : $comment:literal + $( + $reply_chain:tt + )* + ) + )+ + ) => {{ + let (nested_input, _) = mk_comments!( + $( + $( + parent: mk_comments!(@mk_id $name $comment), $name, + $reply_chain + )* + )+ + ); + + let mut input: Vec<RawComment> = vec![ + $( + mk_comments!(@to_raw input $name $comment) + ),+ + ]; + input.extend(nested_input); + + let expected: Comments = Comments { + inner: vec![ + $( + Comment { + value: mk_comments!(@to_raw expected $name $comment), + replies: { + let (_, nested_expected) = mk_comments!( + $( + parent: mk_comments!(@mk_id $name $comment), $name, + $reply_chain + )* + ); + + nested_expected.inner + }, + } + ),+ + ] + }; + + (input, expected) + }}; + + (@mk_id $name:ident $comment:literal) => {{ + use std::hash::{Hash, Hasher}; + + let input = format!("{}{}", stringify!($name), $comment); + + let mut digest = std::hash::DefaultHasher::new(); + input.hash(&mut digest); + Id { id: digest.finish().to_string() } + }}; + + (@to_raw $state:ident $name:ident $comment:literal $($parent:expr, $actual_parent:ident)?) => { + RawComment { + id: mk_comments!(@mk_id $name $comment), + text: mk_comments!(@mk_text $state $comment $(, $actual_parent)?), + like_count: 0, + is_pinned: false, + author_id: stringify!($name).to_owned(), + author: format!("@{}", stringify!($name)), + author_is_verified: false, + author_thumbnail: Url::from_file_path("/dev/null").unwrap(), + parent: mk_comments!(@mk_parent $($parent)?), + edited: false, + timestamp: 0, + author_url: None, + author_is_uploader: false, + is_favorited: false, + } + }; + + (@mk_parent) => { + Parent::Root + }; + (@mk_parent $parent:expr) => { + Parent::Id($parent.id) + }; + + (@mk_text input $text:expr) => { + $text.to_owned() + }; + (@mk_text input $text:expr, $actual_parent:ident) => { + format!("@{} {}", stringify!($actual_parent), $text) + }; + (@mk_text expected $text:expr $(, $_:tt)?) => { + $text.to_owned() + }; +} + +#[test] +fn test_comments_toplevel() { + let (input, expected) = mk_comments!( + (@kant: "I think, that using the results of an action to determine morality is flawed.") + (@hume: "I think, that we should use our feeling for morality more.") + (@lock: "I think, that we should rely on the sum of happiness caused by an action to determine it's morality.") + ); + + assert_eq!(Comments::from_raw(input), expected); +} + +#[test] +fn test_comments_replies_1_level() { + let (input, expected) = mk_comments!( + (@hume: "I think, that we should use our feeling for morality more." + (@kant: "This is so wrong! I shall now dedicate my next 7? years to writing books that prove this.") + (@lock: "It feels not very applicable, no? We should focus on something that can be used in the court of law!")) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_replies_2_levels() { + let (input, expected) = mk_comments!( + (@singer: "We perform medical studies on animals; Children have lower or similar mental ability as these animals.." + (@singer: "Therefore, we should perform these studies on children instead, if we were to follow our own principals" + (@james: "This is ridiculous! I will not entertain this thought.") + (@singer: "Although one could also use this argument to argue for abortion _after_ birth."))) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_replies_3_levels() { + let (input, expected) = mk_comments!( + (@singer: "We perform medical studies on animals; Children have lower or similar mental ability as these animals.." + (@singer: "Therefore, we should perform these studies on children instead, if we were to follow our own principals" + (@james: "This is ridiculous! I will not entertain this thought." + (@singer: "You know that I am not actually suggesting that? This is but a way to critizise the society")) + (@singer: "Although one could also use this argument to argue for abortion _after_ birth."))) + ); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} + +#[test] +fn test_comments_sub_answer_selection() { + let (input, expected) = mk_comments!( + (@coffeewolfproductions9113: "I mean, brothels and sex workers in of themselves are not a bad thing." + (@aikikaname6508: "probably not so much in the 50s, pre contraception") + (@as_ri1mb: "it’s an incredibly sad, degrading line of work, often resulting in self loathing and self-deletion." + (@coffeewolfproductions9113: "Are you speaking from experience?" + (@as_ri1mb: "what an immature response, as expected." + (@coffeewolfproductions9113: "I literally just asked if you were talking from experience."))))) + + ); + + eprintln!("{}", expected.render(true)); + + assert_eq!( + Comments::from_raw(input).render(true), + expected.render(true) + ); +} diff --git a/crates/yt/src/storage/db/video/mod.rs b/crates/yt/src/storage/db/video/mod.rs new file mode 100644 index 0000000..deeb82c --- /dev/null +++ b/crates/yt/src/storage/db/video/mod.rs @@ -0,0 +1,324 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{fmt::Display, path::PathBuf, time::Duration}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::{select::duration::MaybeDuration, storage::db::extractor_hash::ExtractorHash}; + +pub(crate) mod comments; + +macro_rules! video_from_record { + ($record:expr) => { + $crate::storage::db::video::Video { + description: $record.description.clone(), + duration: $crate::select::duration::MaybeDuration::from_maybe_secs_f64( + $record.duration, + ), + extractor_hash: $crate::storage::db::extractor_hash::ExtractorHash::from_hash( + $record + .extractor_hash + .parse() + .expect("The db hash should be a valid blake3 hash"), + ), + last_status_change: $crate::storage::db::video::TimeStamp::from_secs( + $record.last_status_change, + ), + parent_subscription_name: $record.parent_subscription_name.clone(), + publish_date: $record + .publish_date + .map(|pd| $crate::storage::db::video::TimeStamp::from_secs(pd)), + status: { + let marker = + $crate::storage::db::video::VideoStatusMarker::from_db_integer($record.status); + let optional = if let Some(cache_path) = &$record.cache_path { + Some(( + std::path::PathBuf::from(cache_path), + if $record.is_focused == Some(1) { + true + } else { + false + }, + )) + } else { + None + }; + $crate::storage::db::video::VideoStatus::from_marker(marker, optional) + }, + thumbnail_url: if let Some(url) = &$record.thumbnail_url { + Some(url::Url::parse(url).expect("Parsing this as url should always work")) + } else { + None + }, + title: $record.title.clone(), + url: url::Url::parse(&$record.url).expect("Parsing this as url should always work"), + priority: $crate::storage::db::video::Priority::from($record.priority), + watch_progress: std::time::Duration::from_secs( + u64::try_from($record.watch_progress).expect("The record is positive i64"), + ), + subtitle_langs: $record.subtitle_langs.clone(), + playback_speed: $record.playback_speed, + } + }; +} +pub(crate) use video_from_record; + +#[derive(Debug, Clone)] +pub(crate) struct Video { + pub(crate) description: Option<String>, + pub(crate) duration: MaybeDuration, + pub(crate) extractor_hash: ExtractorHash, + pub(crate) last_status_change: TimeStamp, + + /// The associated subscription this video was fetched from (null, when the video was `add`ed) + pub(crate) parent_subscription_name: Option<String>, + pub(crate) priority: Priority, + pub(crate) publish_date: Option<TimeStamp>, + pub(crate) status: VideoStatus, + pub(crate) thumbnail_url: Option<Url>, + pub(crate) title: String, + pub(crate) url: Url, + + /// The seconds the user has already watched the video + pub(crate) watch_progress: Duration, + + /// Which subtitles to include, when downloading this video. + /// In the form of `lang1,lang2,lang3` (e.g. `en,de,sv`) + pub(crate) subtitle_langs: Option<String>, + + /// The playback speed to use, when watching this video. + /// Value is in percent, so 1 is 100%, 2.7 is 270%, and so on. + pub(crate) playback_speed: Option<f64>, +} + +/// The priority of a [`Video`]. +#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct Priority { + value: i64, +} +impl Priority { + /// Return the underlying value to insert that into the database + #[must_use] + pub(crate) fn as_db_integer(self) -> i64 { + self.value + } +} +impl From<i64> for Priority { + fn from(value: i64) -> Self { + Self { value } + } +} +impl Display for Priority { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.value.fmt(f) + } +} + +/// An UNIX time stamp. +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct TimeStamp { + value: i64, +} +impl TimeStamp { + /// Return the seconds since the UNIX epoch for this [`TimeStamp`]. + #[must_use] + pub(crate) fn as_secs(self) -> i64 { + self.value + } + + /// Construct a [`TimeStamp`] from a count of seconds since the UNIX epoch. + #[must_use] + pub(crate) fn from_secs(value: i64) -> Self { + Self { value } + } + + /// Construct a [`TimeStamp`] from the current time. + #[must_use] + pub(crate) fn from_now() -> Self { + Self { + value: Utc::now().timestamp(), + } + } +} +impl Display for TimeStamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + DateTime::from_timestamp(self.value, 0) + .expect("The timestamps should always be valid") + .format("%Y-%m-%d") + .fmt(f) + } +} + +/// # Video Lifetime (words in <brackets> are commands): +/// <Pick> +/// / \ +/// <Watch> <Drop> -> Dropped // yt select +/// | +/// Cache // yt cache +/// | +/// Watched // yt watch +#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub(crate) enum VideoStatus { + #[default] + Pick, + + /// The video has been select to be watched + Watch, + /// The video has been cached and is ready to be watched + Cached { + cache_path: PathBuf, + is_focused: bool, + }, + /// The video has been watched + Watched, + + /// The video has been select to be dropped + Drop, + /// The video has been dropped + Dropped, +} + +impl VideoStatus { + /// Reconstruct a [`VideoStatus`] for it's marker and the optional parts. + /// This should only be used by the db record to [`Video`] code. + /// + /// # Panics + /// Only if internal expectations fail. + #[must_use] + pub(crate) fn from_marker( + marker: VideoStatusMarker, + optional: Option<(PathBuf, bool)>, + ) -> Self { + match marker { + VideoStatusMarker::Pick => Self::Pick, + VideoStatusMarker::Watch => Self::Watch, + VideoStatusMarker::Cached => { + let (cache_path, is_focused) = + optional.expect("This should be some, when the video status is cached"); + Self::Cached { + cache_path, + is_focused, + } + } + VideoStatusMarker::Watched => Self::Watched, + VideoStatusMarker::Drop => Self::Drop, + VideoStatusMarker::Dropped => Self::Dropped, + } + } + + /// Return the associated [`VideoStatusMarker`] for this [`VideoStatus`]. + #[must_use] + pub(crate) fn as_marker(&self) -> VideoStatusMarker { + match self { + VideoStatus::Pick => VideoStatusMarker::Pick, + VideoStatus::Watch => VideoStatusMarker::Watch, + VideoStatus::Cached { .. } => VideoStatusMarker::Cached, + VideoStatus::Watched => VideoStatusMarker::Watched, + VideoStatus::Drop => VideoStatusMarker::Drop, + VideoStatus::Dropped => VideoStatusMarker::Dropped, + } + } +} + +/// Unit only variant of [`VideoStatus`] +#[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] +pub(crate) enum VideoStatusMarker { + #[default] + Pick, + + /// The video has been select to be watched + Watch, + /// The video has been cached and is ready to be watched + Cached, + /// The video has been watched + Watched, + + /// The video has been select to be dropped + Drop, + /// The video has been dropped + Dropped, +} + +impl VideoStatusMarker { + pub(crate) const ALL: &'static [Self; 6] = &[ + Self::Pick, + // + Self::Watch, + Self::Cached, + Self::Watched, + // + Self::Drop, + Self::Dropped, + ]; + + #[must_use] + pub(crate) fn as_command(&self) -> &str { + // NOTE: Keep the serialize able variants synced with the main `select` function <2024-06-14> + // Also try to ensure, that the strings have the same length + match self { + Self::Pick => "pick ", + + Self::Watch | Self::Cached => "watch ", + Self::Watched => "watched", + + Self::Drop | Self::Dropped => "drop ", + } + } + + #[must_use] + pub(crate) fn as_db_integer(self) -> i64 { + // These numbers should not change their mapping! + // Oh, and keep them in sync with the SQLite check constraint. + match self { + Self::Pick => 0, + + Self::Watch => 1, + Self::Cached => 2, + Self::Watched => 3, + + Self::Drop => 4, + Self::Dropped => 5, + } + } + #[must_use] + pub(crate) fn from_db_integer(num: i64) -> Self { + match num { + 0 => Self::Pick, + + 1 => Self::Watch, + 2 => Self::Cached, + 3 => Self::Watched, + + 4 => Self::Drop, + 5 => Self::Dropped, + other => unreachable!( + "The database returned a enum discriminator, unknown to us: '{}'", + other + ), + } + } + + #[must_use] + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Pick => "Pick", + + Self::Watch => "Watch", + Self::Cached => "Cache", + Self::Watched => "Watched", + + Self::Drop => "Drop", + Self::Dropped => "Dropped", + } + } +} diff --git a/crates/yt/src/storage/migrate/mod.rs b/crates/yt/src/storage/migrate/mod.rs new file mode 100644 index 0000000..418c893 --- /dev/null +++ b/crates/yt/src/storage/migrate/mod.rs @@ -0,0 +1,309 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fmt::Display, + future::Future, + time::{SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result, bail}; +use chrono::TimeDelta; +use log::{debug, info}; +use sqlx::{Sqlite, SqlitePool, Transaction, query}; + +use crate::app::App; + +macro_rules! make_upgrade { + ($app:expr, $old_version:expr, $new_version:expr, $sql_name:expr) => { + add_error_context( + async { + let mut tx = $app + .database + .begin() + .await + .context("Failed to start the update transaction")?; + debug!("Migrating: {} -> {}", $old_version, $new_version); + + sqlx::raw_sql(include_str!($sql_name)) + .execute(&mut *tx) + .await + .context("Failed to run the update sql script")?; + + set_db_version( + &mut tx, + if $old_version == Self::Empty { + // There is no previous version we would need to remove + None + } else { + Some($old_version) + }, + $new_version, + ) + .await + .with_context(|| format!("Failed to set the new version ({})", $new_version))?; + + tx.commit() + .await + .context("Failed to commit the update transaction")?; + + // NOTE: This is needed, so that sqlite "sees" our changes to the table + // without having to reconnect. <2025-02-18> + query!("VACUUM") + .execute(&$app.database) + .await + .context("Failed to vacuum database")?; + + Ok(()) + }, + $new_version, + ) + .await?; + + Box::pin($new_version.update($app)).await.context(concat!( + "While updating to version: ", + stringify!($new_version) + )) + }; +} + +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub(crate) enum DbVersion { + /// The database is not yet initialized. + Empty, + + /// The first database version. + /// Introduced: 2025-02-16. + Zero, + + /// Introduced: 2025-02-17. + One, + + /// Introduced: 2025-02-18. + Two, + + /// Introduced: 2025-03-21. + Three, + + /// Introduced: 2025-07-05. + Four, + + /// Introduced: 2025-07-20. + Five, +} +const CURRENT_VERSION: DbVersion = DbVersion::Five; + +async fn add_error_context( + function: impl Future<Output = Result<()>>, + level: DbVersion, +) -> Result<()> { + function + .await + .with_context(|| format!("Failed to migrate database to version: {level}")) +} + +async fn set_db_version( + tx: &mut Transaction<'_, Sqlite>, + old_version: Option<DbVersion>, + new_version: DbVersion, +) -> Result<()> { + let valid_from = get_current_date(); + + if let Some(old_version) = old_version { + let valid_to = valid_from + 1; + let old_version = old_version.as_sql_integer(); + + query!( + "UPDATE version SET valid_to = ? WHERE namespace = 'yt' AND number = ?;", + valid_to, + old_version + ) + .execute(&mut *(*tx)) + .await?; + } + + let version = new_version.as_sql_integer(); + + query!( + "INSERT INTO version (namespace, number, valid_from, valid_to) VALUES ('yt', ?, ?, NULL);", + version, + valid_from + ) + .execute(&mut *(*tx)) + .await?; + + Ok(()) +} + +impl DbVersion { + fn as_sql_integer(self) -> i32 { + match self { + DbVersion::Zero => 0, + DbVersion::One => 1, + DbVersion::Two => 2, + DbVersion::Three => 3, + DbVersion::Four => 4, + DbVersion::Five => 5, + + DbVersion::Empty => unreachable!("A empty version does not have an associated integer"), + } + } + + fn from_db(number: i64, namespace: &str) -> Result<Self> { + match (number, namespace) { + (0, "yt") => Ok(DbVersion::Zero), + (1, "yt") => Ok(DbVersion::One), + (2, "yt") => Ok(DbVersion::Two), + (3, "yt") => Ok(DbVersion::Three), + (4, "yt") => Ok(DbVersion::Four), + (5, "yt") => Ok(DbVersion::Five), + + (0, other) => bail!("Db version is Zero, but got unknown namespace: '{other}'"), + (1, other) => bail!("Db version is One, but got unknown namespace: '{other}'"), + (2, other) => bail!("Db version is Two, but got unknown namespace: '{other}'"), + (3, other) => bail!("Db version is Three, but got unknown namespace: '{other}'"), + (4, other) => bail!("Db version is Four, but got unknown namespace: '{other}'"), + (5, other) => bail!("Db version is Five, but got unknown namespace: '{other}'"), + + (other, "yt") => bail!("Got unkown version for 'yt' namespace: {other}"), + (num, nasp) => bail!("Got unkown version number ({num}) and namespace ('{nasp}')"), + } + } + + /// Try to update the database from version [`self`] to the [`CURRENT_VERSION`]. + /// + /// Each update is atomic, so if this function fails you are still guaranteed to have a + /// database at version `get_version`. + #[allow(clippy::too_many_lines)] + async fn update(self, app: &App) -> Result<()> { + match self { + Self::Empty => { + make_upgrade! {app, Self::Empty, Self::Zero, "./sql/0_Empty_to_Zero.sql"} + } + + Self::Zero => { + make_upgrade! {app, Self::Zero, Self::One, "./sql/1_Zero_to_One.sql"} + } + + Self::One => { + make_upgrade! {app, Self::One, Self::Two, "./sql/2_One_to_Two.sql"} + } + + Self::Two => { + make_upgrade! {app, Self::Two, Self::Three, "./sql/3_Two_to_Three.sql"} + } + + Self::Three => { + make_upgrade! {app, Self::Three, Self::Four, "./sql/4_Three_to_Four.sql"} + } + + Self::Four => { + make_upgrade! {app, Self::Four, Self::Five, "./sql/5_Four_to_Five.sql"} + } + + // This is the current_version + Self::Five => { + assert_eq!(self, CURRENT_VERSION); + assert_eq!(self, get_version(app).await?); + Ok(()) + } + } + } +} +impl Display for DbVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // It is a unit only enum, thus we can simply use the Debug formatting + <Self as std::fmt::Debug>::fmt(self, f) + } +} + +/// Returns the current data as UNIX time stamp. +fn get_current_date() -> i64 { + let start = SystemTime::now(); + let seconds_since_epoch: TimeDelta = TimeDelta::from_std( + start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"), + ) + .expect("Time does not go backwards"); + + // All database dates should be after the UNIX_EPOCH (and thus positiv) + seconds_since_epoch.num_milliseconds() +} + +/// Return the current database version. +/// +/// # Panics +/// Only if internal assertions fail. +pub(crate) async fn get_version(app: &App) -> Result<DbVersion> { + get_version_db(&app.database).await +} + +/// Return the current database version. +/// +/// In contrast to the [`get_version`] function, this function does not +/// a fully instantiated [`App`], a database connection suffices. +/// +/// # Panics +/// Only if internal assertions fail. +pub(crate) async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { + let version_table_exists = { + let query = query!( + " + SELECT 1 as result + FROM sqlite_master + WHERE type = 'table' + AND name = 'version' + " + ) + .fetch_optional(pool) + .await?; + + if let Some(output) = query { + assert_eq!(output.result, 1); + true + } else { + false + } + }; + + if !version_table_exists { + return Ok(DbVersion::Empty); + } + + let current_version = query!( + " + SELECT namespace, number + FROM version + WHERE valid_to IS NULL; + " + ) + .fetch_one(pool) + .await + .context("Failed to fetch version number")?; + + DbVersion::from_db(current_version.number, current_version.namespace.as_str()) +} + +pub(crate) async fn migrate_db(app: &App) -> Result<()> { + let current_version = get_version(app) + .await + .context("Failed to determine initial version")?; + + if current_version == CURRENT_VERSION { + return Ok(()); + } + + info!("Migrate database from version '{current_version}' to version '{CURRENT_VERSION}'"); + + current_version.update(app).await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql new file mode 100644 index 0000000..d703bfc --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql @@ -0,0 +1,72 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +-- All tables should be declared STRICT, as I actually like to have types checking (and a +-- db that doesn't lie to me). + +-- Keep this table in sync with the `DbVersion` enumeration. +CREATE TABLE version ( + -- The `namespace` is only useful, if other tools ever build on this database + namespace TEXT NOT NULL, + + -- The version. + number INTEGER UNIQUE NOT NULL PRIMARY KEY, + + -- The validity of this version as UNIX time stamp + valid_from INTEGER NOT NULL CHECK (valid_from < valid_to), + -- If set to `NULL`, represents the current version + valid_to INTEGER UNIQUE CHECK (valid_to > valid_from) +) STRICT; + +-- Keep this table in sync with the `Video` structure +CREATE TABLE videos ( + cache_path TEXT UNIQUE CHECK (CASE WHEN cache_path IS NOT NULL THEN + status == 2 + ELSE + 1 + END), + description TEXT, + duration REAL, + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + last_status_change INTEGER NOT NULL, + parent_subscription_name TEXT, + priority INTEGER NOT NULL DEFAULT 0, + publish_date INTEGER, + status INTEGER NOT NULL DEFAULT 0 CHECK (status IN (0, 1, 2, 3, 4, 5) AND + CASE WHEN status == 2 THEN + cache_path IS NOT NULL + ELSE + 1 + END AND + CASE WHEN status != 2 THEN + cache_path IS NULL + ELSE + 1 + END), + status_change INTEGER NOT NULL DEFAULT 0 CHECK (status_change IN (0, 1)), + thumbnail_url TEXT, + title TEXT NOT NULL, + url TEXT UNIQUE NOT NULL +) STRICT; + +-- Store additional metadata for the videos marked to be watched +CREATE TABLE video_options ( + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + subtitle_langs TEXT NOT NULL, + playback_speed REAL NOT NULL, + FOREIGN KEY(extractor_hash) REFERENCES videos (extractor_hash) +) STRICT; + +-- Store subscriptions +CREATE TABLE subscriptions ( + name TEXT UNIQUE NOT NULL PRIMARY KEY, + url TEXT NOT NULL +) STRICT; diff --git a/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql new file mode 100644 index 0000000..da9315b --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql @@ -0,0 +1,28 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +-- Is the video currently in a playlist? +ALTER TABLE videos ADD in_playlist INTEGER NOT NULL DEFAULT 0 CHECK (in_playlist IN (0, 1)); +UPDATE videos SET in_playlist = 0; + +-- Is it 'focused' (i.e., the select video)? +-- Only of video should be focused at a time. +ALTER TABLE videos +ADD COLUMN is_focused INTEGER NOT NULL DEFAULT 0 +CHECK (is_focused IN (0, 1)); +UPDATE videos SET is_focused = 0; + +-- The progress the user made in watching the video. +ALTER TABLE videos ADD watch_progress INTEGER NOT NULL DEFAULT 0 CHECK (watch_progress <= duration); +-- Assume, that the user has watched the video to end, if it is marked as watched +UPDATE videos SET watch_progress = ifnull(duration, 0) WHERE status = 3; +UPDATE videos SET watch_progress = 0 WHERE status != 3; + +ALTER TABLE videos DROP COLUMN status_change; diff --git a/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql new file mode 100644 index 0000000..806de07 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql @@ -0,0 +1,11 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +ALTER TABLE videos DROP in_playlist; diff --git a/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql new file mode 100644 index 0000000..b33f849 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql @@ -0,0 +1,85 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + + +-- 1. Create new table +-- 2. Copy data +-- 3. Drop old table +-- 4. Rename new into old + +-- remove the original TRANSACTION +COMMIT TRANSACTION; + +-- tweak config +PRAGMA foreign_keys=OFF; + +-- start your own TRANSACTION +BEGIN TRANSACTION; + +CREATE TABLE videos_new ( + cache_path TEXT UNIQUE CHECK (CASE + WHEN cache_path IS NOT NULL THEN status == 2 + ELSE 1 + END), + description TEXT, + duration REAL, + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + last_status_change INTEGER NOT NULL, + parent_subscription_name TEXT, + priority INTEGER NOT NULL DEFAULT 0, + publish_date INTEGER, + status INTEGER NOT NULL DEFAULT 0 CHECK (status IN (0, 1, 2, 3, 4, 5) AND + CASE + WHEN status == 2 THEN cache_path IS NOT NULL + WHEN status != 2 THEN cache_path IS NULL + ELSE 1 + END), + thumbnail_url TEXT, + title TEXT NOT NULL, + url TEXT UNIQUE NOT NULL, + is_focused INTEGER UNIQUE DEFAULT NULL CHECK (CASE + WHEN is_focused IS NOT NULL THEN is_focused == 1 + ELSE 1 + END), + watch_progress INTEGER NOT NULL DEFAULT 0 CHECK (watch_progress <= duration) +) STRICT; + +INSERT INTO videos_new SELECT + videos.cache_path, + videos.description, + videos.duration, + videos.extractor_hash, + videos.last_status_change, + videos.parent_subscription_name, + videos.priority, + videos.publish_date, + videos.status, + videos.thumbnail_url, + videos.title, + videos.url, + dummy.is_focused, + videos.watch_progress +FROM videos, (SELECT NULL AS is_focused) AS dummy; + +DROP TABLE videos; + +ALTER TABLE videos_new RENAME TO videos; + +-- check foreign key constraint still upholding. +PRAGMA foreign_key_check; + +-- commit your own TRANSACTION +COMMIT TRANSACTION; + +-- rollback all config you setup before. +PRAGMA foreign_keys=ON; + +-- start a new TRANSACTION to let migrator commit it. +BEGIN TRANSACTION; diff --git a/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql b/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql new file mode 100644 index 0000000..9c283a1 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/4_Three_to_Four.sql @@ -0,0 +1,24 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +ALTER TABLE videos +ADD COLUMN subtitle_langs TEXT; + +ALTER TABLE videos +ADD COLUMN playback_speed REAL CHECK (playback_speed >= 0); + +UPDATE videos + SET playback_speed = video_options.playback_speed, + subtitle_langs = video_options.subtitle_langs + FROM video_options + WHERE videos.extractor_hash = video_options.extractor_hash; + + +DROP TABLE video_options; diff --git a/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql b/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql new file mode 100644 index 0000000..6c4b7cc --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/5_Four_to_Five.sql @@ -0,0 +1,15 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + + +CREATE TABLE txn_log ( + timestamp INTEGER NOT NULL, + operation TEXT NOT NULL +) STRICT; diff --git a/crates/yt_dlp/src/wrapper/mod.rs b/crates/yt/src/storage/mod.rs index 3fe3247..6dcff74 100644 --- a/crates/yt_dlp/src/wrapper/mod.rs +++ b/crates/yt/src/storage/mod.rs @@ -1,6 +1,7 @@ // yt - A fully featured command line YouTube client // // Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // // This file is part of Yt. @@ -8,5 +9,6 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -pub mod info_json; -// pub mod yt_dlp_options; +pub(crate) mod db; +pub(crate) mod migrate; +pub(crate) mod notify; diff --git a/crates/yt/src/storage/notify.rs b/crates/yt/src/storage/notify.rs new file mode 100644 index 0000000..e0ee4e9 --- /dev/null +++ b/crates/yt/src/storage/notify.rs @@ -0,0 +1,77 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + path::{Path, PathBuf}, + sync::mpsc, + thread::sleep, + time::Duration, +}; + +use crate::app::App; + +use anyhow::{Context, Result}; +use notify::{ + Event, EventKind, RecursiveMode, Watcher, + event::{DataChange, ModifyKind}, +}; +use tokio::task; + +/// This functions registers a watcher for the database and only returns once a write was +/// registered for the database. +pub(crate) async fn wait_for_db_write(app: &App) -> Result<()> { + let db_path: PathBuf = app.config.paths.database_path.clone(); + task::spawn_blocking(move || wait_for_db_write_sync(&db_path)).await? +} + +fn wait_for_db_write_sync(db_path: &Path) -> Result<()> { + let (tx, rx) = mpsc::channel::<notify::Result<Event>>(); + + let mut watcher = notify::recommended_watcher(tx)?; + + watcher.watch(db_path, RecursiveMode::NonRecursive)?; + + for res in rx { + let event = res.context("Failed to wait for db write")?; + + if let EventKind::Modify(ModifyKind::Data(DataChange::Any)) = event.kind { + // Buffer some of the `Modify` event burst. + sleep(Duration::from_millis(10)); + + return Ok(()); + } + } + + Ok(()) +} + +/// This functions registers a watcher for the cache path and returns once a file was removed +pub(crate) async fn wait_for_cache_reduction(app: &App) -> Result<()> { + let download_directory: PathBuf = app.config.paths.download_dir.clone(); + task::spawn_blocking(move || wait_for_cache_reduction_sync(&download_directory)).await? +} + +fn wait_for_cache_reduction_sync(download_directory: &Path) -> Result<()> { + let (tx, rx) = mpsc::channel::<notify::Result<Event>>(); + + let mut watcher = notify::recommended_watcher(tx)?; + + watcher.watch(download_directory, RecursiveMode::Recursive)?; + + for res in rx { + let event = res.context("Failed to wait for cache size reduction")?; + + if let EventKind::Remove(_) = event.kind { + return Ok(()); + } + } + + Ok(()) +} diff --git a/crates/yt/src/version/mod.rs b/crates/yt/src/version/mod.rs new file mode 100644 index 0000000..b12eadd --- /dev/null +++ b/crates/yt/src/version/mod.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::{Context, Result}; +use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; +use yt_dlp::options::YoutubeDLOptions; + +use crate::{config::Config, storage::migrate::get_version_db}; + +pub(crate) async fn show(config: &Config) -> Result<()> { + let db_version = { + let options = SqliteConnectOptions::new() + .filename(&config.paths.database_path) + .optimize_on_close(true, None) + .create_if_missing(true); + + let pool = SqlitePool::connect_with(options) + .await + .context("Failed to connect to database!")?; + + get_version_db(&pool) + .await + .context("Failed to determine database version")? + }; + + let (yt_dlp, python) = { + let yt_dlp = YoutubeDLOptions::new().build()?; + yt_dlp.version()? + }; + + let python = python.replace('\n', " "); + + println!( + "{}: {} + +db version: {db_version} + +yt-dlp: {yt_dlp} +python: {python}", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_VERSION"), + ); + + Ok(()) +} diff --git a/crates/yt/src/videos/format_video.rs b/crates/yt/src/videos/format_video.rs new file mode 100644 index 0000000..6598780 --- /dev/null +++ b/crates/yt/src/videos/format_video.rs @@ -0,0 +1,133 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use colors::Colorize; + +use crate::{app::App, output::format_text, storage::db::video::Video, videos::RenderWithApp}; + +impl Video { + pub(crate) async fn to_info_display( + &self, + app: &App, + format: Option<String>, + ) -> Result<String> { + let cache_path = self.cache_path_fmt().to_string(app); + let description = self.description_fmt().to_string(app); + let duration = self.duration_fmt().to_string(app); + let extractor_hash = self.extractor_hash_fmt(app).await?.to_string(app); + let in_playlist = self.in_playlist_fmt().to_string(app); + let last_status_change = self.last_status_change_fmt().to_string(app); + let parent_subscription_name = self.parent_subscription_name_fmt().to_string(app); + let priority = self.priority_fmt().to_string(app); + let publish_date = self.publish_date_fmt().to_string(app); + let status = self.status_fmt().to_string(app); + let thumbnail_url = self.thumbnail_url_fmt().to_string(app); + let title = self.title_fmt().to_string(app); + let url = self.url_fmt().to_string(app); + let video_options = self.video_options_fmt(app).to_string(app); + + let watched_percentage_fmt = { + if let Some(percent) = self.watch_progress_percent_fmt() { + format!(" (watched: {})", percent.to_string(app)) + } else { + format!(" {}", self.watch_progress_fmt().to_string(app)) + } + }; + + let options = video_options.to_string(); + let options = options.trim(); + let description = format_text(description.to_string().as_str(), None); + + let string = if let Some(format) = format { + format + .replace("{title}", &title) + .replace("{extractor_hash}", &extractor_hash) + .replace("{cache_path}", &cache_path) + .replace("{duration}", &duration) + .replace("{watched_percentage_fmt}", &watched_percentage_fmt) + .replace("{parent_subscription_name}", &parent_subscription_name) + .replace("{priority}", &priority) + .replace("{publish_date}", &publish_date) + .replace("{status}", &status) + .replace("{last_status_change}", &last_status_change) + .replace("{in_playlist}", &in_playlist) + .replace("{thumbnail_url}", &thumbnail_url) + .replace("{url}", &url) + .replace("{options}", options) + .replace("{description}", &description) + } else { + format!( + "\ +{title} ({extractor_hash}) +| -> {cache_path} +| -> {duration}{watched_percentage_fmt} +| -> {parent_subscription_name} +| -> priority: {priority} +| -> {publish_date} +| -> status: {status} since {last_status_change} ({in_playlist}) +| -> {thumbnail_url} +| -> {url} +| -> options: {options} +{description}\n", + ) + }; + Ok(string) + } + + pub(crate) async fn to_line_display( + &self, + app: &App, + format: Option<String>, + ) -> Result<String> { + let status = self.status_fmt().to_string(app); + let extractor_hash = self.extractor_hash_fmt(app).await?.to_string(app); + let title = self.title_fmt().to_string(app); + let publish_date = self.publish_date_fmt().to_string(app); + let parent_subscription_name = self.parent_subscription_name_fmt().to_string(app); + let duration = self.duration_fmt().to_string(app); + let url = self.url_fmt().to_string(app); + + let f = if let Some(format) = format { + format + .replace("{status}", &status) + .replace("{extractor_hash}", &extractor_hash) + .replace("{title}", &title) + .replace("{publish_date}", &publish_date) + .replace("{parent_subscription_name}", &parent_subscription_name) + .replace("{duration}", &duration) + .replace("{url}", &url) + } else { + format!( + "{status} {extractor_hash} {title} {publish_date} {parent_subscription_name} {duration}" + ) + }; + + Ok(f) + } + + pub(crate) async fn to_select_file_display(&self, app: &App) -> Result<String> { + let f = format!( + r#"{}{} {} "{}" "{}" "{}" "{}" "{}"{}"#, + self.status_fmt().render(false), + self.video_options_fmt(app).render(false), + self.extractor_hash_fmt(app).await?.render(false), + self.title_fmt().render(false), + self.publish_date_fmt().render(false), + self.parent_subscription_name_fmt().render(false), + self.duration_fmt().render(false), + self.url_fmt().render(false), + '\n' + ); + + Ok(f) + } +} diff --git a/crates/yt/src/videos/mod.rs b/crates/yt/src/videos/mod.rs new file mode 100644 index 0000000..c2f01fa --- /dev/null +++ b/crates/yt/src/videos/mod.rs @@ -0,0 +1,213 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::Write; + +use anyhow::{Context, Result}; +use colors::{Colorize, IntoCanvas}; +use url::Url; + +use crate::{ + app::App, + select::duration::MaybeDuration, + storage::db::video::{TimeStamp, Video, VideoStatus}, +}; + +pub(crate) mod format_video; + +macro_rules! get { + ($value:expr, $key:ident, $name:expr, $code:tt) => { + if let Some(value) = &$value.$key { + $code(value) + } else { + concat!("[No ", $name, "]").to_owned() + } + }; +} + +pub(crate) trait RenderWithApp: Colorize { + fn to_string(self, app: &App) -> String { + self.render(app.config.global.display_colors) + } +} +impl<C: Colorize> RenderWithApp for C {} + +impl Video { + #[must_use] + pub(crate) fn cache_path_fmt(&self) -> impl Colorize { + let cache_path = if let VideoStatus::Cached { + cache_path, + is_focused: _, + } = &self.status + { + cache_path.to_string_lossy().to_string() + } else { + "[No Cache Path]".to_owned() + }; + + cache_path.blue().bold() + } + + #[must_use] + pub(crate) fn description_fmt(&self) -> impl Colorize { + get!( + self, + description, + "Description", + (|value: &str| value.to_owned()) + ) + .into_canvas() + } + + #[must_use] + pub(crate) fn duration_fmt(&self) -> impl Colorize { + self.duration.cyan().bold() + } + + #[must_use] + pub(crate) fn watch_progress_fmt(&self) -> impl Colorize { + MaybeDuration::from_std(self.watch_progress).cyan().bold() + } + #[must_use] + pub(crate) fn watch_progress_percent_fmt(&self) -> Option<impl Colorize> { + self.duration.as_secs_f64().map(|duration| { + let watch_progress = self.watch_progress.as_secs_f64(); + + (format!("{:0.0}%", (watch_progress / duration) * 100.0)).into_canvas() + }) + } + + pub(crate) async fn extractor_hash_fmt(&self, app: &App) -> Result<impl Colorize> { + let hash = self + .extractor_hash + .as_short_hash(app) + .await + .with_context(|| { + format!( + "Failed to format extractor hash, whilst formatting video: '{}'", + self.title + ) + })?; + + Ok(hash.purple().bold().italic()) + } + + #[must_use] + pub(crate) fn in_playlist_fmt(&self) -> impl Colorize { + let output = match &self.status { + VideoStatus::Pick + | VideoStatus::Watch + | VideoStatus::Watched + | VideoStatus::Drop + | VideoStatus::Dropped => "Not in the playlist", + VideoStatus::Cached { is_focused, .. } => { + if *is_focused { + "In the playlist and focused" + } else { + "In the playlist" + } + } + }; + output.yellow().italic() + } + #[must_use] + pub(crate) fn last_status_change_fmt(&self) -> impl Colorize { + self.last_status_change.bright_cyan() + } + + #[must_use] + pub(crate) fn parent_subscription_name_fmt(&self) -> impl Colorize { + let psn = get!( + self, + parent_subscription_name, + "author", + (|sub: &str| sub.replace('"', "'")) + ); + + psn.bright_magenta() + } + + #[must_use] + pub(crate) fn priority_fmt(&self) -> impl Colorize { + self.priority.into_canvas() + } + + #[must_use] + pub(crate) fn publish_date_fmt(&self) -> impl Colorize { + let date = get!( + self, + publish_date, + "release date", + (|date: &TimeStamp| date.to_string()) + ); + + date.bright_white().bold() + } + + #[must_use] + pub(crate) fn status_fmt(&self) -> impl Colorize { + // TODO: We might support `.trim()`ing that, as the extra whitespace could be bad in the + // selection file. <2024-10-07> + let status = self.status.as_marker().as_command().to_owned(); + + status.red().bold() + } + + #[must_use] + pub(crate) fn thumbnail_url_fmt(&self) -> impl Colorize { + get!( + self, + thumbnail_url, + "thumbnail URL", + (|url: &Url| url.to_string()) + ) + .into_canvas() + } + + #[must_use] + pub(crate) fn title_fmt(&self) -> impl Colorize { + let title = self.title.replace(['"', '„', '”', '“'], "'"); + + title.green().bold() + } + + #[must_use] + pub(crate) fn url_fmt(&self) -> impl Colorize { + let url = self.url.as_str().replace('"', "\\\""); + + url.italic() + } + + pub(crate) fn video_options_fmt(&self, app: &App) -> impl Colorize { + let video_options = { + let mut opts = String::new(); + + if let Some(playback_speed) = self.playback_speed { + if (playback_speed - app.config.select.playback_speed).abs() > f64::EPSILON { + write!(opts, " --playback-speed '{}'", playback_speed).expect("In-memory"); + } + } + + if let Some(subtitle_langs) = &self.subtitle_langs { + if subtitle_langs != &app.config.select.subtitle_langs { + write!(opts, " --subtitle-langs '{}'", subtitle_langs).expect("In-memory"); + } + } + + let opts = opts.trim().to_owned(); + + let opts_white = if opts.is_empty() { "" } else { " " }; + format!("{opts_white}{opts}") + }; + + video_options.bright_green() + } +} diff --git a/crates/yt/src/yt_dlp/mod.rs b/crates/yt/src/yt_dlp/mod.rs new file mode 100644 index 0000000..eaa80a1 --- /dev/null +++ b/crates/yt/src/yt_dlp/mod.rs @@ -0,0 +1,253 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{borrow::ToOwned, str::FromStr, time::Duration}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use futures::{FutureExt, future::BoxFuture}; +use log::{error, warn}; +use serde_json::json; +use tokio::{fs, io}; +use url::Url; +use yt_dlp::{ + YoutubeDL, info_json::InfoJson, json_cast, json_get, json_try_get, options::YoutubeDLOptions, +}; + +use crate::{ + app::App, + select::duration::MaybeDuration, + shared::bytes::Bytes, + storage::db::{ + extractor_hash::ExtractorHash, + subscription::Subscription, + video::{Priority, TimeStamp, Video, VideoStatus}, + }, +}; + +pub(crate) fn yt_dlp_opts_updating(max_backlog: usize) -> Result<YoutubeDL> { + Ok(YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", max_backlog) + .set("noplaylist", false) + .set( + "extractor_args", + json! {{"youtubetab": {"approximate_date": [""]}}}, + ) + // // TODO: This also removes unlisted and other stuff. Find a good way to remove the + // // members-only videos from the feed. <2025-04-17> + // .set("match-filter", "availability=public") + .build()?) +} + +impl Video { + pub(crate) fn get_approx_size(&self) -> Result<u64> { + let yt_dlp = { + YoutubeDLOptions::new() + .set("prefer_free_formats", true) + .set("format", "bestvideo[height<=?1080]+bestaudio/best") + .set("fragment_retries", 10) + .set("retries", 10) + .set("getcomments", false) + .set("ignoreerrors", false) + .build() + .context("Failed to instanciate get approx size yt_dlp") + }?; + + let result = yt_dlp + .extract_info(&self.url, false, true) + .with_context(|| format!("Failed to extract video information: '{}'", self.title))?; + + let size = if let Some(filesize) = json_try_get!(result, "filesize", as_u64) { + filesize + } else if let Some(num) = json_try_get!(result, "filesize_approx", as_u64) { + num + } else if let Some(duration) = json_try_get!(result, "duration", as_f64) + && let Some(tbr) = json_try_get!(result, "tbr", as_f64) + { + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let duration = duration.ceil() as u64; + + // TODO: yt_dlp gets this from the format + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let tbr = tbr.ceil() as u64; + + duration * tbr * (1000 / 8) + } else { + let hardcoded_default = Bytes::from_str("250 MiB").expect("This is hardcoded"); + error!( + "Failed to find a filesize for video: {:?} (Using hardcoded value of {})", + self.title, hardcoded_default + ); + hardcoded_default.as_u64() + }; + + Ok(size) + } +} + +impl Video { + #[allow(clippy::too_many_lines)] + pub(crate) fn from_info_json(entry: &InfoJson, sub: Option<&Subscription>) -> Result<Video> { + fn fmt_context(date: &str, extended: Option<&str>) -> String { + let f = format!( + "Failed to parse the `upload_date` of the entry ('{date}'). \ + Expected `YYYY-MM-DD`, has the format changed?" + ); + if let Some(date_string) = extended { + format!("{f}\nThe parsed '{date_string}' can't be turned to a valid UTC date.'") + } else { + f + } + } + + let publish_date = if let Some(date) = json_try_get!(entry, "upload_date", as_str) { + let year: u32 = date + .chars() + .take(4) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let month: u32 = date + .chars() + .skip(4) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let day: u32 = date + .chars() + .skip(4 + 2) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + + let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z"); + Some( + DateTime::<Utc>::from_str(&date_string) + .with_context(|| fmt_context(date, Some(&date_string)))? + .timestamp(), + ) + } else { + warn!( + "The video '{}' lacks it's upload date!", + json_get!(entry, "title", as_str) + ); + None + }; + + let thumbnail_url = match ( + &json_try_get!(entry, "thumbnails", as_array), + &json_try_get!(entry, "thumbnail", as_str), + ) { + (None, None) => None, + (None, Some(thumbnail)) => Some(Url::from_str(thumbnail)?), + + // TODO: The algorithm is not exactly the best <2024-05-28> + (Some(thumbnails), None) => { + if let Some(thumbnail) = thumbnails.first() { + Some(Url::from_str(json_get!( + json_cast!(thumbnail, as_object), + "url", + as_str + ))?) + } else { + None + } + } + (Some(_), Some(thumnail)) => Some(Url::from_str(thumnail)?), + }; + + let url = { + let smug_url: Url = json_get!(entry, "webpage_url", as_str).parse()?; + // TODO(@bpeetz): We should probably add this? <2025-06-14> + // if '#__youtubedl_smuggle' not in smug_url: + // return smug_url, default + // url, _, sdata = smug_url.rpartition('#') + // jsond = urllib.parse.parse_qs(sdata)['__youtubedl_smuggle'][0] + // data = json.loads(jsond) + // return url, data + + smug_url + }; + + let extractor_hash = ExtractorHash::from_info_json(entry); + + let subscription_name = if let Some(sub) = sub { + Some(sub.name.clone()) + } else if let Some(uploader) = json_try_get!(entry, "uploader", as_str) { + if json_try_get!(entry, "webpage_url_domain", as_str) == Some("youtube.com") { + Some(format!("{uploader} - Videos")) + } else { + Some(uploader.to_owned()) + } + } else { + None + }; + + let video = Video { + description: json_try_get!(entry, "description", as_str).map(ToOwned::to_owned), + duration: MaybeDuration::from_maybe_secs_f64(json_try_get!(entry, "duration", as_f64)), + extractor_hash, + last_status_change: TimeStamp::from_now(), + parent_subscription_name: subscription_name, + priority: Priority::default(), + publish_date: publish_date.map(TimeStamp::from_secs), + status: VideoStatus::Pick, + thumbnail_url, + title: json_get!(entry, "title", as_str).to_owned(), + url, + watch_progress: Duration::default(), + playback_speed: None, + subtitle_langs: None, + }; + Ok(video) + } +} + +pub(crate) async fn get_current_cache_allocation(app: &App) -> Result<Bytes> { + fn dir_size(mut dir: fs::ReadDir) -> BoxFuture<'static, Result<Bytes>> { + async move { + let mut acc = 0; + while let Some(entry) = dir.next_entry().await? { + let size = match entry.metadata().await? { + data if data.is_dir() => { + let path = entry.path(); + let read_dir = fs::read_dir(path).await?; + + dir_size(read_dir).await?.as_u64() + } + data => data.len(), + }; + acc += size; + } + Ok(Bytes::new(acc)) + } + .boxed() + } + + let read_dir_result = match fs::read_dir(&app.config.paths.download_dir).await { + Ok(ok) => ok, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => { + unreachable!("The download dir should always be created in the config finalizers."); + } + err => Err(io::Error::from(err)).with_context(|| { + format!( + "Failed to get dir size of download dir at: '{}'", + &app.config.paths.download_dir.display() + ) + })?, + }, + }; + + dir_size(read_dir_result).await +} diff --git a/crates/yt/tests/_testenv/init.rs b/crates/yt/tests/_testenv/init.rs new file mode 100644 index 0000000..5970c7c --- /dev/null +++ b/crates/yt/tests/_testenv/init.rs @@ -0,0 +1,136 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + env, + fs::{self, OpenOptions}, + io::{self, Write}, + path::PathBuf, +}; + +use crate::{_testenv::Paths, testenv::TestEnv}; + +fn target_dir() -> PathBuf { + // Tests exe is in target/debug/deps, the *yt* exe is in target/debug + env::current_exe() + .expect("./target/debug/deps/yt-*") + .parent() + .expect("./target/debug/deps") + .parent() + .expect("./target/debug") + .parent() + .expect("./target") + .to_path_buf() +} + +fn test_dir(name: &'static str) -> PathBuf { + target_dir().join("tests").join(name) +} + +fn prepare_files_and_dirs(name: &'static str) -> io::Result<Paths> { + let test_dir = test_dir(name); + + fs::create_dir_all(&test_dir)?; + + let db_path = test_dir.join("database.sqlite"); + let last_selection_path = test_dir.join("last_selection"); + let config_path = test_dir.join("config.toml"); + let download_dir = test_dir.join("download"); + + { + // Remove all files, that are not the download dir. + for entry in fs::read_dir(test_dir).expect("Works") { + let entry = entry.unwrap(); + let entry_ft = entry.file_type().unwrap(); + + if entry_ft.is_dir() && entry.file_name() == "download" { + continue; + } + + if entry_ft.is_dir() { + fs::remove_dir_all(entry.path()).unwrap(); + } else { + fs::remove_file(entry.path()).unwrap(); + } + } + } + + fs::create_dir_all(&download_dir)?; + + { + let mut config_file = OpenOptions::new() + .write(true) + .truncate(true) + .create(true) + .open(&config_path)?; + + writeln!( + config_file, + r#"[paths] +download_dir = "{}" +mpv_config_path = "/dev/null" +mpv_input_path = "/dev/null" +database_path = "{}" +last_selection_path = "{}" + +[download] +max_cache_size = "100GiB" + +[update] +max_backlog = 1 +"#, + download_dir.display(), + db_path.display(), + last_selection_path.display(), + )?; + config_file.flush()?; + } + + Ok(Paths { + db: db_path, + last_selection: last_selection_path, + config: config_path, + download_dir, + }) +} + +/// Find the *yt* executable. +fn find_yt_exe() -> PathBuf { + let target = target_dir().join("debug"); + + let exe_name = if cfg!(windows) { "yt.exe" } else { "yt" }; + + target.join(exe_name) +} + +impl TestEnv { + pub(crate) fn new(name: &'static str) -> TestEnv { + let yt_exe = find_yt_exe(); + let test_dir = test_dir(name); + + let paths = prepare_files_and_dirs(name).expect("config dir"); + + TestEnv { + name, + yt_exe, + test_dir, + paths, + spawned_childs: vec![], + } + } +} + +impl Drop for TestEnv { + fn drop(&mut self) { + for child in &mut self.spawned_childs { + drop(child.kill()); + } + } +} diff --git a/crates/yt/tests/_testenv/mod.rs b/crates/yt/tests/_testenv/mod.rs new file mode 100644 index 0000000..38d1f0a --- /dev/null +++ b/crates/yt/tests/_testenv/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! This code was taken from *fd* at 30-06-2025. + +use std::{path::PathBuf, process}; + +mod init; +mod run; +pub(crate) mod util; + +/// Environment for the integration tests. +pub(crate) struct TestEnv { + pub(crate) name: &'static str, + pub(crate) yt_exe: PathBuf, + pub(crate) test_dir: PathBuf, + + pub(crate) paths: Paths, + + pub(crate) spawned_childs: Vec<process::Child>, +} + +pub(crate) struct Paths { + pub(crate) db: PathBuf, + pub(crate) last_selection: PathBuf, + pub(crate) config: PathBuf, + pub(crate) download_dir: PathBuf, +} diff --git a/crates/yt/tests/_testenv/run.rs b/crates/yt/tests/_testenv/run.rs new file mode 100644 index 0000000..578d823 --- /dev/null +++ b/crates/yt/tests/_testenv/run.rs @@ -0,0 +1,183 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::HashMap, + process::{self, Stdio}, +}; + +use colors::{Colorize, IntoCanvas}; + +use crate::testenv::TestEnv; + +/// Format an error message for when *yt* did not exit successfully. +fn format_exit_error(args: &[&str], output: &process::Output) -> String { + format!( + "`yt {}` did not exit successfully.\nstdout:\n---\n{}---\nstderr:\n---\n{}---", + args.join(" "), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// Format an error message for when the output of *yt* did not match the expected output. +fn format_output_error(args: &[&str], expected: &str, actual: &str) -> String { + fn normalize_str(input: &str) -> &str { + if input.is_empty() { "<Empty>" } else { input } + } + + let expected = normalize_str(expected); + let actual = normalize_str(actual); + + format!( + concat!( + "`yt {}` did not produce the expected output.\n", + "expected:\n---\n{}\n---\n and actual:\n---\n{}\n---\n" + ), + args.join(" "), + expected, + actual + ) +} + +/// Normalize the output for comparison. +fn normalize_output(s: &str, trim_start: bool, normalize_line: bool) -> String { + // Split into lines and normalize separators. + let mut lines = s + .replace('\0', "NULL\n") + .lines() + .map(|line| { + let line = if trim_start { line.trim_start() } else { line }; + let line = line.replace('/', std::path::MAIN_SEPARATOR_STR); + if normalize_line { + let mut words: Vec<_> = line.split_whitespace().collect(); + words.sort_unstable(); + return words.join(" "); + } + line + }) + .collect::<Vec<_>>(); + + lines.sort(); + lines.join("\n") +} + +impl TestEnv { + /// Assert that calling *yt* in the specified path under the root working directory, + /// and with the specified arguments produces the expected output. + pub(crate) fn assert_output(&self, args: &[&str], expected: &str) { + let expected = normalize_output(expected, true, false); + let actual = self.run(args); + + assert!( + expected == actual, + "{}", + format_output_error(args, &expected, &actual) + ); + } + + /// Run *yt* once with the first args, pipe the output of this command to *yt* with the second + /// args and return the normalized output. + pub(crate) fn run_piped(&self, first_args: &[&str], second_args: &[&str]) -> String { + let mut first_cmd = self.prepare_yt(first_args); + let mut second_cmd = self.prepare_yt(second_args); + + first_cmd.stdout(Stdio::piped()); + let mut first_child = first_cmd.spawn().expect("yt spawn"); + + second_cmd.stdin(first_child.stdout.take().expect("Was set")); + + let first_output = first_child.wait_with_output().expect("yt run"); + assert!( + first_output.status.success(), + "{}", + format_exit_error(first_args, &first_output) + ); + + Self::finalize_cmd(second_cmd, second_args) + } + + /// Run *yt*, with the given args. + /// Returns the normalized stdout output. + pub(crate) fn run(&self, args: &[&str]) -> String { + let cmd = self.prepare_yt(args); + Self::finalize_cmd(cmd, args) + } + + /// Run *yt*, with the given args and environment variables. + /// Returns the normalized stdout output. + pub(crate) fn run_env(&self, args: &[&str], env: &HashMap<&str, &str>) -> String { + let mut cmd = self.prepare_yt(args); + cmd.envs(env.iter()); + Self::finalize_cmd(cmd, args) + } + + /// Run *yt*, with the given args and fork into the background. + /// Returns a sender for the lines on stdout. + pub(crate) fn run_background(&mut self, args: &[&str]) -> process::ChildStdout { + let mut cmd = self.prepare_yt(args); + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::inherit()); + + // The whole point of this function, is calling a command and keep it running for the whole + // programs run-time. + // + // And we provide a clean-up mechanism. + #[allow(clippy::zombie_processes)] + let mut child = cmd.spawn().expect("yt spawn"); + + let stdout = child.stdout.take().expect("Was piped"); + + self.spawned_childs.push(child); + + stdout + } + + fn finalize_cmd(mut cmd: process::Command, args: &[&str]) -> String { + let child = cmd.spawn().expect("yt spawn"); + let output = child.wait_with_output().expect("yt output"); + + assert!( + output.status.success(), + "{}", + format_exit_error(args, &output) + ); + + normalize_output(&String::from_utf8_lossy(&output.stdout), false, false) + } + + fn prepare_yt(&self, args: &[&str]) -> process::Command { + let mut cmd = process::Command::new(&self.yt_exe); + + cmd.current_dir(&self.test_dir); + + cmd.args([ + "-v", + "--color", + "false", + "--config-path", + self.paths.config.to_str().unwrap(), + ]); + + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + cmd.args(args); + + eprintln!( + "{} `yt {}`", + self.name.blue().italic().render(true), + args.join(" ") + ); + + cmd + } +} diff --git a/crates/yt/tests/_testenv/util.rs b/crates/yt/tests/_testenv/util.rs new file mode 100644 index 0000000..6633fbf --- /dev/null +++ b/crates/yt/tests/_testenv/util.rs @@ -0,0 +1,371 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::_testenv::TestEnv; + +use std::{ + collections::HashMap, + fs::{OpenOptions, Permissions}, + io::Write, + os::unix::fs::PermissionsExt, + path::PathBuf, +}; + +pub(crate) fn get_first_hash(env: &TestEnv) -> String { + let output = env.run(&["videos", "ls", "--format", "{extractor_hash}"]); + + let first_hash = output.lines().next().unwrap(); + first_hash.to_owned() +} + +#[derive(Clone, Copy)] +pub(crate) enum Subscription { + Tagesschau, +} + +impl Subscription { + const fn as_url(self) -> &'static str { + match self { + Subscription::Tagesschau => { + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU" + } + } + } + + const fn as_name(self) -> &'static str { + match self { + Subscription::Tagesschau => "Tagesschau", + } + } +} + +/// Provide the given number of videos. +pub(crate) fn provide_videos(env: &TestEnv, sub: Subscription, num: u8) { + add_sub(env, sub); + update_sub(env, num, sub); +} + +pub(crate) fn add_sub(env: &TestEnv, sub: Subscription) { + env.run(&[ + "subs", + "add", + sub.as_url(), + "--name", + sub.as_name(), + "--no-check", + ]); +} + +pub(crate) fn update_sub(env: &TestEnv, num_of_videos: u8, sub: Subscription) { + let num_of_videos: &str = u8_to_char(num_of_videos); + + env.run(&["update", "--max-backlog", num_of_videos, sub.as_name()]); +} + +pub(crate) fn run_select(env: &TestEnv, sed_regex: &str) { + let mut map = HashMap::new(); + + let command = make_command( + env, + format!(r#"sed --in-place '{sed_regex}' "$1""#).as_str(), + ); + map.insert("EDITOR", command.to_str().unwrap()); + + env.run_env(&["select", "file", "--done"], &map); +} + +fn make_command(env: &TestEnv, shell_command: &str) -> PathBuf { + let command_path = env.test_dir.join("command.sh"); + + { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&command_path) + .unwrap(); + + writeln!(file, "#!/usr/bin/env sh").unwrap(); + + file.write_all(shell_command.as_bytes()).unwrap(); + file.flush().unwrap(); + + { + let perms = Permissions::from_mode(0o0700); + file.set_permissions(perms).unwrap(); + } + } + + command_path +} + +// Char conversion {{{ +#[allow(clippy::too_many_lines)] +const fn u8_to_char(input: u8) -> &'static str { + match input { + 0 => "0", + 1 => "1", + 2 => "2", + 3 => "3", + 4 => "4", + 5 => "5", + 6 => "6", + 7 => "7", + 8 => "8", + 9 => "9", + 10 => "10", + 11 => "11", + 12 => "12", + 13 => "13", + 14 => "14", + 15 => "15", + 16 => "16", + 17 => "17", + 18 => "18", + 19 => "19", + 20 => "20", + 21 => "21", + 22 => "22", + 23 => "23", + 24 => "24", + 25 => "25", + 26 => "26", + 27 => "27", + 28 => "28", + 29 => "29", + 30 => "30", + 31 => "31", + 32 => "32", + 33 => "33", + 34 => "34", + 35 => "35", + 36 => "36", + 37 => "37", + 38 => "38", + 39 => "39", + 40 => "40", + 41 => "41", + 42 => "42", + 43 => "43", + 44 => "44", + 45 => "45", + 46 => "46", + 47 => "47", + 48 => "48", + 49 => "49", + 50 => "50", + 51 => "51", + 52 => "52", + 53 => "53", + 54 => "54", + 55 => "55", + 56 => "56", + 57 => "57", + 58 => "58", + 59 => "59", + 60 => "60", + 61 => "61", + 62 => "62", + 63 => "63", + 64 => "64", + 65 => "65", + 66 => "66", + 67 => "67", + 68 => "68", + 69 => "69", + 70 => "70", + 71 => "71", + 72 => "72", + 73 => "73", + 74 => "74", + 75 => "75", + 76 => "76", + 77 => "77", + 78 => "78", + 79 => "79", + 80 => "80", + 81 => "81", + 82 => "82", + 83 => "83", + 84 => "84", + 85 => "85", + 86 => "86", + 87 => "87", + 88 => "88", + 89 => "89", + 90 => "90", + 91 => "91", + 92 => "92", + 93 => "93", + 94 => "94", + 95 => "95", + 96 => "96", + 97 => "97", + 98 => "98", + 99 => "99", + 100 => "100", + 101 => "101", + 102 => "102", + 103 => "103", + 104 => "104", + 105 => "105", + 106 => "106", + 107 => "107", + 108 => "108", + 109 => "109", + 110 => "110", + 111 => "111", + 112 => "112", + 113 => "113", + 114 => "114", + 115 => "115", + 116 => "116", + 117 => "117", + 118 => "118", + 119 => "119", + 120 => "120", + 121 => "121", + 122 => "122", + 123 => "123", + 124 => "124", + 125 => "125", + 126 => "126", + 127 => "127", + 128 => "128", + 129 => "129", + 130 => "130", + 131 => "131", + 132 => "132", + 133 => "133", + 134 => "134", + 135 => "135", + 136 => "136", + 137 => "137", + 138 => "138", + 139 => "139", + 140 => "140", + 141 => "141", + 142 => "142", + 143 => "143", + 144 => "144", + 145 => "145", + 146 => "146", + 147 => "147", + 148 => "148", + 149 => "149", + 150 => "150", + 151 => "151", + 152 => "152", + 153 => "153", + 154 => "154", + 155 => "155", + 156 => "156", + 157 => "157", + 158 => "158", + 159 => "159", + 160 => "160", + 161 => "161", + 162 => "162", + 163 => "163", + 164 => "164", + 165 => "165", + 166 => "166", + 167 => "167", + 168 => "168", + 169 => "169", + 170 => "170", + 171 => "171", + 172 => "172", + 173 => "173", + 174 => "174", + 175 => "175", + 176 => "176", + 177 => "177", + 178 => "178", + 179 => "179", + 180 => "180", + 181 => "181", + 182 => "182", + 183 => "183", + 184 => "184", + 185 => "185", + 186 => "186", + 187 => "187", + 188 => "188", + 189 => "189", + 190 => "190", + 191 => "191", + 192 => "192", + 193 => "193", + 194 => "194", + 195 => "195", + 196 => "196", + 197 => "197", + 198 => "198", + 199 => "199", + 200 => "200", + 201 => "201", + 202 => "202", + 203 => "203", + 204 => "204", + 205 => "205", + 206 => "206", + 207 => "207", + 208 => "208", + 209 => "209", + 210 => "210", + 211 => "211", + 212 => "212", + 213 => "213", + 214 => "214", + 215 => "215", + 216 => "216", + 217 => "217", + 218 => "218", + 219 => "219", + 220 => "220", + 221 => "221", + 222 => "222", + 223 => "223", + 224 => "224", + 225 => "225", + 226 => "226", + 227 => "227", + 228 => "228", + 229 => "229", + 230 => "230", + 231 => "231", + 232 => "232", + 233 => "233", + 234 => "234", + 235 => "235", + 236 => "236", + 237 => "237", + 238 => "238", + 239 => "239", + 240 => "240", + 241 => "241", + 242 => "242", + 243 => "243", + 244 => "244", + 245 => "245", + 246 => "246", + 247 => "247", + 248 => "248", + 249 => "249", + 250 => "250", + 251 => "251", + 252 => "252", + 253 => "253", + 254 => "254", + 255 => "255", + } +} +// }}} diff --git a/crates/yt/tests/select/base.rs b/crates/yt/tests/select/base.rs new file mode 100644 index 0000000..24e198b --- /dev/null +++ b/crates/yt/tests/select/base.rs @@ -0,0 +1,50 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, fs}; + +use crate::{ + _testenv::{ + TestEnv, + util::{self, Subscription}, + }, + select::get_videos_in_state, +}; + +#[test] +fn test_base() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, Subscription::Tagesschau, 4); + + let first_hash = &util::get_first_hash(&env); + env.run(&["select", "drop", first_hash]); + + let mut map = HashMap::new(); + map.insert("EDITOR", "true"); + env.run_env(&["select", "file", "--done"], &map); + + fs::remove_file(&env.paths.last_selection).unwrap(); + + env.run_env(&["select", "split", "--done"], &map); + assert_states(&env); + + env.run_env(&["select", "file", "--use-last-selection"], &map); + assert_states(&env); +} + +fn assert_states(env: &TestEnv) { + assert_eq!(get_videos_in_state(env, "Picked"), 3); + assert_eq!(get_videos_in_state(env, "Drop"), 1); + + assert_eq!(get_videos_in_state(env, "Watch"), 0); + assert_eq!(get_videos_in_state(env, "Cached"), 0); + assert_eq!(get_videos_in_state(env, "Watched"), 0); +} diff --git a/crates/yt/tests/select/file.rs b/crates/yt/tests/select/file.rs new file mode 100644 index 0000000..b8bd2b5 --- /dev/null +++ b/crates/yt/tests/select/file.rs @@ -0,0 +1,31 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + _testenv::util::{self, Subscription}, + select::get_videos_in_state, + testenv::TestEnv, +}; + +#[test] +fn test_file() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, Subscription::Tagesschau, 4); + + util::run_select(&env, "s/pick/watch/"); + + assert_eq!(get_videos_in_state(&env, "Picked"), 0); + assert_eq!(get_videos_in_state(&env, "Drop"), 0); + + assert_eq!(get_videos_in_state(&env, "Watch"), 4); + assert_eq!(get_videos_in_state(&env, "Cached"), 0); + assert_eq!(get_videos_in_state(&env, "Watched"), 0); +} diff --git a/crates/yt/tests/select/mod.rs b/crates/yt/tests/select/mod.rs new file mode 100644 index 0000000..d7033f8 --- /dev/null +++ b/crates/yt/tests/select/mod.rs @@ -0,0 +1,25 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::_testenv::TestEnv; + +mod base; +mod file; +mod options; + +fn get_videos_in_state(env: &TestEnv, state: &str) -> usize { + let status = env.run(&[ + "status", + "--format", + format!("{{{}_videos_len}}", state.to_lowercase()).as_str(), + ]); + + status.parse().unwrap() +} diff --git a/crates/yt/tests/select/options.rs b/crates/yt/tests/select/options.rs new file mode 100644 index 0000000..6a0d155 --- /dev/null +++ b/crates/yt/tests/select/options.rs @@ -0,0 +1,51 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{_testenv::util, select::get_videos_in_state, testenv::TestEnv}; + +#[test] +fn test_options() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, util::Subscription::Tagesschau, 1); + + let video_hash = &util::get_first_hash(&env); + env.run(&["select", "watch", video_hash]); + + env.assert_output(&["videos", "info", video_hash, "--format", "{options}"], ""); + + env.run(&[ + "select", + "watch", + video_hash, + "--playback-speed", + "1", + "--subtitle-langs", + "en,de,sv", + ]); + + env.assert_output( + &["videos", "info", video_hash, "--format", "{options}"], + "--playback-speed '1' --subtitle-langs 'en,de,sv'", + ); + + env.run(&["select", "watch", video_hash, "-s", "1.7", "-l", "de"]); + + env.assert_output( + &["videos", "info", video_hash, "--format", "{options}"], + "--playback-speed '1.7' --subtitle-langs 'de'", + ); + + assert_eq!(get_videos_in_state(&env, "Picked"), 0); + assert_eq!(get_videos_in_state(&env, "Drop"), 0); + assert_eq!(get_videos_in_state(&env, "Watch"), 1); + assert_eq!(get_videos_in_state(&env, "Cached"), 0); + assert_eq!(get_videos_in_state(&env, "Watched"), 0); +} diff --git a/crates/yt/tests/subscriptions/import_export/golden.txt b/crates/yt/tests/subscriptions/import_export/golden.txt new file mode 100644 index 0000000..7ed5419 --- /dev/null +++ b/crates/yt/tests/subscriptions/import_export/golden.txt @@ -0,0 +1,2 @@ +Nyheter på lätt svenska: 'https://www.svtplay.se/nyheter-pa-latt-svenska' +tagesschau: 'https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU' diff --git a/crates/yt/tests/subscriptions/import_export/golden.txt.license b/crates/yt/tests/subscriptions/import_export/golden.txt.license new file mode 100644 index 0000000..7813eb6 --- /dev/null +++ b/crates/yt/tests/subscriptions/import_export/golden.txt.license @@ -0,0 +1,9 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +SPDX-License-Identifier: GPL-3.0-or-later + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/crates/yt/tests/subscriptions/import_export/mod.rs b/crates/yt/tests/subscriptions/import_export/mod.rs new file mode 100644 index 0000000..1156508 --- /dev/null +++ b/crates/yt/tests/subscriptions/import_export/mod.rs @@ -0,0 +1,35 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::testenv::TestEnv; + +#[test] +fn test_import_export() { + let env = TestEnv::new(module_path!()); + + env.run(&[ + "subs", + "add", + "https://www.svtplay.se/nyheter-pa-latt-svenska", + ]); + env.run(&[ + "subs", + "add", + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU", + ]); + + let before = env.run(&["subs", "list"]); + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); + + env.run_piped(&["subs", "export"], &["subs", "import", "--force"]); + + env.assert_output(&["subs", "list"], &before); + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); +} diff --git a/crates/yt/tests/subscriptions/mod.rs b/crates/yt/tests/subscriptions/mod.rs new file mode 100644 index 0000000..0b300c5 --- /dev/null +++ b/crates/yt/tests/subscriptions/mod.rs @@ -0,0 +1,12 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +mod import_export; +mod naming_subscriptions; diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt new file mode 100644 index 0000000..46ede50 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt @@ -0,0 +1,2 @@ +Nyheter: 'https://www.svtplay.se/nyheter-pa-latt-svenska' +Vewn: 'https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU' diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license new file mode 100644 index 0000000..7813eb6 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/golden.txt.license @@ -0,0 +1,9 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +SPDX-License-Identifier: GPL-3.0-or-later + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs b/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs new file mode 100644 index 0000000..50fe3e4 --- /dev/null +++ b/crates/yt/tests/subscriptions/naming_subscriptions/mod.rs @@ -0,0 +1,33 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::testenv::TestEnv; + +#[test] +fn test_naming_subscriptions() { + let env = TestEnv::new(module_path!()); + + env.run(&[ + "subs", + "add", + "https://www.svtplay.se/nyheter-pa-latt-svenska", + "--name", + "Nyheter", + ]); + env.run(&[ + "subs", + "add", + "https://www.ardmediathek.de/sendung/tagesschau/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhZ2Vzc2NoYXU", + "--name", + "Vewn", + ]); + + env.assert_output(&["subs", "list"], include_str!("./golden.txt")); +} diff --git a/crates/yt/tests/tests.rs b/crates/yt/tests/tests.rs new file mode 100644 index 0000000..89c3091 --- /dev/null +++ b/crates/yt/tests/tests.rs @@ -0,0 +1,22 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// Use this, for the background run pids +// #![feature(linux_pidfd)] + +#![allow(unused_crate_dependencies)] + +mod _testenv; +pub(crate) use _testenv as testenv; + +mod select; +mod subscriptions; +mod videos; +mod watch; diff --git a/crates/yt/tests/videos/downloading.rs b/crates/yt/tests/videos/downloading.rs new file mode 100644 index 0000000..f026858 --- /dev/null +++ b/crates/yt/tests/videos/downloading.rs @@ -0,0 +1,52 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{_testenv::util, testenv::TestEnv}; + +#[test] +fn test_downloading() { + let env = TestEnv::new(module_path!()); + + util::provide_videos(&env, util::Subscription::Tagesschau, 1); + + let first_hash = &util::get_first_hash(&env); + env.run(&["select", "watch", first_hash]); + + env.run(&["download"]); + + let usage = get_cache_usage(&env); + assert!(usage > 0.0); + + env.run(&["cache", "clear"]); + + let usage = get_cache_usage(&env); + + #[allow(clippy::float_cmp)] + { + assert_eq!(usage, 0.0); + } +} + +fn get_cache_usage(env: &TestEnv) -> f64 { + let status = env.run(&["status", "--format", "{cache_usage}"]); + + let split: Vec<_> = status.split(' ').collect(); + let usage: f64 = split[0].parse().unwrap(); + let unit = split[1]; + + #[allow(clippy::cast_precision_loss)] + match unit { + "B" => usage * (1024u64.pow(0)) as f64, + "KiB" => usage * (1024u64.pow(1)) as f64, + "MiB" => usage * (1024u64.pow(2)) as f64, + "GiB" => usage * (1024u64.pow(3)) as f64, + other => unreachable!("Unknown unit: {other}"), + } +} diff --git a/crates/yt/tests/videos/mod.rs b/crates/yt/tests/videos/mod.rs new file mode 100644 index 0000000..6a80761 --- /dev/null +++ b/crates/yt/tests/videos/mod.rs @@ -0,0 +1,11 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +mod downloading; diff --git a/crates/yt/tests/watch/focus_switch.rs b/crates/yt/tests/watch/focus_switch.rs new file mode 100644 index 0000000..81246f3 --- /dev/null +++ b/crates/yt/tests/watch/focus_switch.rs @@ -0,0 +1,53 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use yt_dlp::json_cast; + +use crate::{_testenv::util, testenv::TestEnv, watch::MpvControl}; + +#[test] +#[ignore = "Currently, this test is missing it's goal"] +fn test_focus_switch() { + let mut env = TestEnv::new(module_path!()); + + { + util::provide_videos(&env, util::Subscription::Tagesschau, 32); + + util::run_select(&env, "s/pick/watch/"); + + env.run(&["download"]); + } + + let mut mpv = MpvControl::new(&mut env); + + assert_pos(&mut mpv, 0); + + for i in 1..32 { + mpv.assert(&["playlist-next", "weak"]); + assert_pos(&mut mpv, i); + } + + mpv.assert(&["playlist-next", "weak"]); + assert_pos(&mut mpv, 2); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 1); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 0); + + mpv.assert(&["playlist-prev", "weak"]); + assert_pos(&mut mpv, 0); +} + +fn assert_pos(mpv: &mut MpvControl, pos: i64) { + let mpv_pos = mpv.assert(&["get_property", "playlist-pos"]); + assert_eq!(json_cast!(mpv_pos, as_i64), pos); +} diff --git a/crates/yt/tests/watch/mod.rs b/crates/yt/tests/watch/mod.rs new file mode 100644 index 0000000..7af8b39 --- /dev/null +++ b/crates/yt/tests/watch/mod.rs @@ -0,0 +1,135 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + io::{BufRead, BufReader, Write}, + os::unix::net::UnixStream, + path::PathBuf, + sync::atomic::AtomicU64, +}; + +use colors::{Colorize, IntoCanvas}; +use serde_json::json; +use yt_dlp::{json_cast, json_get, json_try_get}; + +use crate::_testenv::TestEnv; + +mod focus_switch; + +struct MpvControl { + stream: UnixStream, + current_request_id: AtomicU64, + name: &'static str, +} + +impl MpvControl { + fn new(env: &mut TestEnv) -> Self { + let socket_path = { + let stdout = env.run_background(&[ + "watch", + // "--headless", + "--provide-ipc-socket", + ]); + + let line = { + let mut buf = String::new(); + let mut reader = BufReader::new(stdout); + reader.read_line(&mut buf).expect("In-memory"); + buf + }; + + PathBuf::from(line.trim()) + }; + + let stream = UnixStream::connect(&socket_path).unwrap_or_else(|e| { + panic!( + "Path to socket ('{}') should exist, but did not: {e}", + socket_path.display() + ) + }); + + let mut me = Self { + stream, + name: env.name, + current_request_id: AtomicU64::new(0), + }; + + // Disable all events. + // We do not use them, and this should reduce the read load on the socket. + me.assert(&["disable_event", "all"]); + + me + } + + /// Run a command and assert that it ran successfully. + fn assert(&mut self, args: &[&str]) -> serde_json::Value { + let out = self.command(args); + + out.unwrap_or_else(|e| panic!("`mpv {}` failed; error {e}.", args.join(" "))) + } + + /// Run a command in mpv. + /// Will return true if the command ran correctly and false if not. + fn command(&mut self, args: &[&str]) -> Result<serde_json::Value, String> { + let tl_rid = self + .current_request_id + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + eprint!( + "{} `mpv {}`", + self.name.blue().italic().render(true), + args.join(" ") + ); + + writeln!( + self.stream, + "{}", + json!( { "command": args, "request_id": tl_rid }) + ) + .expect("Should always work"); + + loop { + let response: serde_json::Value = { + let mut reader = BufReader::new(&mut self.stream); + + let mut buf = String::new(); + reader.read_line(&mut buf).expect("Works"); + serde_json::from_str(&buf).expect("Mpv only returns json") + }; + + if let Some(rid) = json_try_get!(response, "request_id", as_u64) { + if rid == tl_rid { + let error = json_get!(response, "error", as_str); + + if error == "success" { + let data: serde_json::Value = { + match response.get("data") { + Some(val) => val.to_owned(), + None => serde_json::Value::Null, + } + }; + + eprintln!(", {}: {data}", "output".bright_blue().render(true),); + return Ok(data); + } + + eprintln!(", {}: {error}", "error".bright_red().render(true)); + return Err(error.to_owned()); + } + } + } + } +} + +impl Drop for MpvControl { + fn drop(&mut self) { + self.assert(&["quit"]); + } +} diff --git a/crates/yt_dlp/Cargo.toml b/crates/yt_dlp/Cargo.toml index 1d34371..87bb610 100644 --- a/crates/yt_dlp/Cargo.toml +++ b/crates/yt_dlp/Cargo.toml @@ -10,7 +10,7 @@ [package] name = "yt_dlp" -description = "A wrapper around the python yt_dlp library" +description = "A rust ffi wrapper library for the python yt_dlp library" keywords = [] categories = [] version.workspace = true @@ -19,19 +19,18 @@ authors.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true -publish = false +publish = true [dependencies] -pyo3 = { version = "0.23.3", features = ["auto-initialize"] } -bytes.workspace = true +curl = "0.4.48" log.workspace = true -serde.workspace = true +pyo3 = { workspace = true } +pyo3-pylogger = { path = "crates/pyo3-pylogger" } +serde = { workspace = true, features = ["derive"] } serde_json.workspace = true +thiserror = "2.0.12" url.workspace = true -[dev-dependencies] -tokio.workspace = true - [lints] workspace = true diff --git a/crates/yt_dlp/README.md b/crates/yt_dlp/README.md index 591ef2e..ece8540 100644 --- a/crates/yt_dlp/README.md +++ b/crates/yt_dlp/README.md @@ -12,7 +12,7 @@ If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. # Yt_py -> \[can be empty\] +> [can be empty] Some text about the project. diff --git a/crates/yt_dlp/.cargo/config.toml b/crates/yt_dlp/crates/pyo3-pylogger/.gitignore index d84f14d..733c5bc 100644 --- a/crates/yt_dlp/.cargo/config.toml +++ b/crates/yt_dlp/crates/pyo3-pylogger/.gitignore @@ -1,12 +1,13 @@ # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 # # This file is part of Yt. # # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -[env] -PYO3_PYTHON = "/nix/store/7xzk119acyws2c4ysygdv66l0grxkr39-python3-3.11.9-env/bin/python3" +target +Cargo.lock +.idea diff --git a/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml b/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml new file mode 100644 index 0000000..28dfacd --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/Cargo.toml @@ -0,0 +1,31 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +[package] +name = "pyo3-pylogger" +version = "1.8.0" +edition = "2021" +authors = [ + "Dylan Bobby Storey <dylan.storey@gmail.com>", + "cpu <daniel@binaryparadox.net>", + "Warren Snipes <contact@warrensnipes.dev>", +] +description = "Enables `log` for pyo3 based Rust applications using the `logging` modules." +publish = ["crates-io"] +license = "Apache-2.0" +readme = "README.md" +homepage = "https://github.com/dylanbstorey/pyo3-pylogger" +repository = "https://github.com/dylanbstorey/pyo3-pylogger" +documentation = "https://github.com/dylanbstorey/pyo3-pylogger" + +[dependencies] +pyo3 = { workspace = true } +log = { workspace = true } +phf = { version = "0.12", features = ["macros"] } diff --git a/crates/yt_dlp/crates/pyo3-pylogger/LICENSE b/crates/yt_dlp/crates/pyo3-pylogger/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/yt_dlp/crates/pyo3-pylogger/README.md b/crates/yt_dlp/crates/pyo3-pylogger/README.md new file mode 100644 index 0000000..e68903b --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/README.md @@ -0,0 +1,160 @@ +<!-- +yt - A fully featured command line YouTube client + +Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +SPDX-License-Identifier: Apache-2.0 + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. +--> + +# pyo3-pylogger + +Enables log messages for pyo3 embedded Python applications using Python's +`logging` or module. + +# Features + +- Logging integration between Python's `logging` module and Rust's `log` crate +- Structured logging support via the logging + [extra](https://docs.python.org/3/library/logging.html#logging.Logger.debug) + field (requires `kv` or `tracing-kv`feature) +- Integration with Rust's `tracing` library (requires `tracing` feature) + +# Usage + +```rust +use log::{info, warn}; +use pyo3::{ffi::c_str, prelude::*}; +fn main() { + // register the host handler with python logger, providing a logger target + pyo3_pylogger::register("example_application_py_logger"); + + // initialize up a logger + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace")).init(); + //just show the logger working from Rust. + info!("Just some normal information!"); + warn!("Something spooky happened!"); + + // Ask pyo3 to set up embedded Python interpreter + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + // Python code can now `import logging` as usual + py.run( + c_str!( + r#" +import logging +logging.getLogger().setLevel(0) +logging.debug('DEBUG') +logging.info('INFO') +logging.warning('WARNING') +logging.error('ERROR') +logging.getLogger('foo.bar.baz').info('INFO')"# + ), + None, + None, + ) + .unwrap(); + }) +} + + +``` + +## Outputs + +```bash +[2025-03-28T01:12:29Z INFO helloworld] Just some normal information! +[2025-03-28T01:12:29Z WARN helloworld] Something spooky happened! +[2025-03-28T01:12:29Z DEBUG example_application_py_logger] DEBUG +[2025-03-28T01:12:29Z INFO example_application_py_logger] INFO +[2025-03-28T01:12:29Z WARN example_application_py_logger] WARNING +[2025-03-28T01:12:29Z ERROR example_application_py_logger] ERROR +[2025-03-28T01:12:29Z INFO example_application_py_logger::foo::bar::baz] INFO +``` + +## Structured Logging + +To enable structured logging support, add the `kv` feature to your `Cargo.toml`: + +```toml +[dependencies] +pyo3-pylogger = { version = "0.4", features = ["kv"] } +``` + +Then you can use Python's `extra` parameter to pass structured data: + +```python +logging.info("Processing order", extra={"order_id": "12345", "amount": 99.99}) +``` + +When using a structured logging subscriber in Rust, these key-value pairs will +be properly captured, for example: + +```bash +[2025-03-28T01:12:29Z INFO example_application_py_logger] Processing order order_id=12345 amount=99.99 +``` + +## Tracing Support + +To enable integration with Rust's `tracing` library, add the `tracing` feature +to your `Cargo.toml`: + +```toml +[dependencies] +pyo3-pylogger = { version = "0.4", default-features = false, features = ["tracing"] } +``` + +When the `tracing` feature is enabled, Python logs will be forwarded to the +active tracing subscriber: + +```rust +use tracing::{info, warn}; +use pyo3::{ffi::c_str, prelude::*}; + +fn main() { + // Register the tracing handler with Python logger + pyo3_pylogger::register_tracing("example_application_py_logger"); + + // Initialize tracing subscriber + tracing_subscriber::fmt::init(); + + // Tracing events from Rust + info!("Tracing information from Rust"); + + // Python logging will be captured by the tracing subscriber + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + py.run( + c_str!( + r#" +import logging +logging.getLogger().setLevel(0) +logging.info('This will be captured by tracing')"# + ), + None, + None, + ) + .unwrap(); + }) +} +``` + +### Structured Data with Tracing + +The `tracing` feature automatically supports Python's `extra` field for +structured data. However, the KV fields are json serialized and not available as +tracing attributes. This is a limitation of the `tracing` library and is not +specific to this crate. See +[this issue](https://github.com/tokio-rs/tracing/issues/372) for more +information. + +# Feature Flags + +- `kv`: Enables structured logging support via Python's `extra` fields. This + adds support for the `log` crate's key-value system. +- `tracing`: Enables integration with Rust's `tracing` library. +- `tracing-kv`: Enables structured logging support via Python's `extra` fields + and integration with Rust's `tracing` library. diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs new file mode 100644 index 0000000..67a0c3e --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/kv.rs @@ -0,0 +1,127 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! Key-Value handling module for Python LogRecord attributes. +//! +//! This module provides functionality to extract and handle custom key-value pairs +//! from Python LogRecord objects, facilitating integration between Python's logging +//! system and Rust's log crate. + +use pyo3::{ + Bound, PyAny, PyResult, + types::{PyAnyMethods, PyDict, PyDictMethods, PyListMethods}, +}; +use std::collections::HashMap; + +/// A static hashset containing all standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes defined in the CPython logging module. +/// +/// This set is used to differentiate between standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes and custom key-value pairs +/// that users might add to their log records. The attributes listed here correspond to the default +/// attributes created by Python's [makeRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L1633-L1634) function. +pub static LOG_RECORD_KV_ATTRIBUTES: phf::Set<&'static str> = phf::phf_set! { + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "taskName", +}; + +/// Extracts custom key-value pairs from a Python LogRecord object. +/// +/// This function examines the `__dict__` of a LogRecord(https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) object and identifies any attributes +/// that are not part of the standard [LogRecord](https://github.com/python/cpython/blob/8a00c9a4d2ce9d373b13f8f0a2265a65f4523293/Lib/logging/__init__.py#L286-L287) attributes. These custom attributes are +/// treated as key-value pairs for structured logging. +/// +/// # Arguments +/// * `record` - A reference to a Python LogRecord object +/// +/// # Returns +/// * `PyResult<Option<HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>>>` - If custom attributes +/// are found, returns a HashMap containing the key-value pairs. Returns None if no custom +/// attributes are present. +/// +/// # Note +/// This function relies on the fact that Python will not implement new attributes on the LogRecord object. +/// If new attributes are added, this function will not be able to filter them out and will return them as key-value pairs. +/// In that future, [LOG_RECORD_KV_ATTRIBUTES] will need to be updated to include the new attributes. +/// This is an unfortunate side effect of using the `__dict__` attribute to extract key-value pairs. However, there are no other ways to handle this given that CPython does not distinguish between user-provided attributes and attributes created by the logging module. +pub fn find_kv_args<'a>( + record: &Bound<'a, PyAny>, +) -> PyResult<Option<std::collections::HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>>> { + let dict: Bound<'_, PyDict> = record.getattr("__dict__")?.extract()?; + + // We can abuse the fact that Python dictionaries are ordered by insertion order to reverse iterate over the keys + // and stop at the first key that is not a predefined key-value pair attribute. + let mut kv_args: Option<HashMap<String, pyo3::Bound<'_, pyo3::PyAny>>> = None; + + for item in dict.items().iter().rev() { + let (key, value) = + item.extract::<(pyo3::Bound<'_, pyo3::PyAny>, pyo3::Bound<'_, pyo3::PyAny>)>()?; + + let key_str = key.to_string(); + if LOG_RECORD_KV_ATTRIBUTES.contains(&key_str) { + break; + } + if kv_args.is_none() { + kv_args = Some(HashMap::new()); + } + + kv_args.as_mut().unwrap().insert(key_str, value); + } + + Ok(kv_args) +} + +/// A wrapper struct that implements the `log::kv::Source` trait for Python key-value pairs. +/// +/// This struct allows Python LogRecord custom attributes to be used with Rust's +/// structured logging system by implementing the necessary trait for key-value handling. +/// +/// # Type Parameters +/// * `'a` - The lifetime of the contained Python values +pub struct KVSource<'a>(pub HashMap<String, pyo3::Bound<'a, pyo3::PyAny>>); + +impl log::kv::Source for KVSource<'_> { + /// Visits each key-value pair in the source, converting Python values to debug representations. + /// + /// # Arguments + /// * `visitor` - The visitor that will process each key-value pair + /// + /// # Returns + /// * `Result<(), log::kv::Error>` - Success if all pairs are visited successfully, + /// or an error if visitation fails + fn visit<'kvs>( + &'kvs self, + visitor: &mut dyn log::kv::VisitSource<'kvs>, + ) -> Result<(), log::kv::Error> { + for (key, value) in &self.0 { + let v: log::kv::Value<'_> = log::kv::Value::from_debug(value); + + visitor.visit_pair(log::kv::Key::from_str(key), v)?; + } + Ok(()) + } +} diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs new file mode 100644 index 0000000..d244ef4 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/level.rs @@ -0,0 +1,43 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +/// A wrapper type for logging levels that supports both `tracing` and `log` features. +pub(crate) struct Level(pub log::Level); + +/// Converts a numeric level value to the appropriate logging Level. +/// +/// # Arguments +/// +/// * `level` - A u8 value representing the logging level: +/// * 40+ = Error +/// * 30-39 = Warn +/// * 20-29 = Info +/// * 10-19 = Debug +/// * 0-9 = Trace +/// +/// # Returns +/// +/// Returns a `Level` wrapper containing either a `tracing::Level` or `log::Level` +/// depending on which feature is enabled. +pub(crate) fn get_level(level: u8) -> Level { + { + if level.ge(&40u8) { + Level(log::Level::Error) + } else if level.ge(&30u8) { + Level(log::Level::Warn) + } else if level.ge(&20u8) { + Level(log::Level::Info) + } else if level.ge(&10u8) { + Level(log::Level::Debug) + } else { + Level(log::Level::Trace) + } + } +} diff --git a/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs b/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs new file mode 100644 index 0000000..3ecb123 --- /dev/null +++ b/crates/yt_dlp/crates/pyo3-pylogger/src/lib.rs @@ -0,0 +1,211 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +// SPDX-License-Identifier: Apache-2.0 +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + ffi::CString, + sync::{self, OnceLock}, +}; + +use log::{debug, log_enabled}; +use pyo3::{ + Bound, Py, PyAny, PyResult, Python, pyfunction, + sync::OnceLockExt, + types::{PyAnyMethods, PyDict, PyListMethods, PyModuleMethods}, + wrap_pyfunction, +}; + +mod kv; +mod level; + +static LOGGER: sync::OnceLock<Py<PyAny>> = OnceLock::new(); + +/// Is the specified record to be logged? Returns false for no, +/// true for yes. Filters can either modify log records in-place or +/// return a completely different record instance which will replace +/// the original log record in any future processing of the event. +#[pyfunction] +fn filter_error_log<'py>(record: Bound<'py, PyAny>) -> bool { + // Filter out all error logs (they are propagated as rust errors) + let levelname: String = record + .getattr("levelname") + .expect("This should exist") + .extract() + .expect("This should be a String"); + + let return_value = levelname.as_str() != "ERROR"; + + if log_enabled!(log::Level::Debug) && !return_value { + let message: String = { + let get_message = record.getattr("getMessage").expect("Is set"); + let message: String = get_message + .call((), None) + .expect("Can be called") + .extract() + .expect("Downcasting works"); + + message.as_str().to_owned() + }; + + debug!("Swollowed error message: '{message}'"); + } + return_value +} + +/// Consume a Python `logging.LogRecord` and emit a Rust `Log` instead. +#[pyfunction] +fn host_log(record: Bound<'_, PyAny>, rust_target: &str) -> PyResult<()> { + let level = record.getattr("levelno")?.extract()?; + let message = record.getattr("getMessage")?.call0()?.to_string(); + let pathname = record.getattr("pathname")?.extract::<String>()?; + let lineno = record.getattr("lineno")?.extract::<u32>()?; + + let logger_name = record.getattr("name")?.extract::<String>()?; + + let full_target: Option<String> = if logger_name.trim().is_empty() || logger_name == "root" { + None + } else { + // Libraries (ex: tracing_subscriber::filter::Directive) expect rust-style targets like foo::bar, + // and may not deal well with "." as a module separator: + let logger_name = logger_name.replace('.', "::"); + Some(format!("{rust_target}::{logger_name}")) + }; + let target = full_target.as_deref().unwrap_or(rust_target); + + handle_record(record, target, &message, lineno, &pathname, level)?; + + Ok(()) +} + +fn handle_record( + #[allow(unused_variables)] record: Bound<'_, PyAny>, + target: &str, + message: &str, + lineno: u32, + pathname: &str, + level: u8, +) -> PyResult<()> { + // If log feature is enabled, use log::logger + let level = crate::level::get_level(level).0; + + { + let mut metadata_builder = log::MetadataBuilder::new(); + metadata_builder.target(target); + metadata_builder.level(level); + + let mut record_builder = log::Record::builder(); + + { + let kv_args = kv::find_kv_args(&record)?; + + let kv_source = kv_args.map(kv::KVSource); + if let Some(kv_source) = kv_source { + log::logger().log( + &record_builder + .metadata(metadata_builder.build()) + .args(format_args!("{}", &message)) + .line(Some(lineno)) + .file(Some(pathname)) + .module_path(Some(pathname)) + .key_values(&kv_source) + .build(), + ); + return Ok(()); + } + } + + log::logger().log( + &record_builder + .metadata(metadata_builder.build()) + .args(format_args!("{}", &message)) + .line(Some(lineno)) + .file(Some(pathname)) + .module_path(Some(pathname)) + .build(), + ); + } + + Ok(()) +} + +/// Registers the host_log function in rust as the event handler for Python's logging logger +/// This function needs to be called from within a pyo3 context as early as possible to ensure logging messages +/// arrive to the rust consumer. +pub fn setup_logging<'py>(py: Python<'py>, target: &str) -> PyResult<Bound<'py, PyAny>> { + let logger = LOGGER + .get_or_init_py_attached(py, || match setup_logging_inner(py, target) { + Ok(ok) => ok.unbind(), + Err(err) => { + panic!("Failed to initialize logger: {}", err); + } + }) + .clone_ref(py); + + Ok(logger.into_bound(py)) +} + +fn setup_logging_inner<'py>(py: Python<'py>, target: &str) -> PyResult<Bound<'py, PyAny>> { + let logging = py.import("logging")?; + + logging.setattr("host_log", wrap_pyfunction!(host_log, &logging)?)?; + + #[allow(clippy::uninlined_format_args)] + let code = CString::new(format!( + r#" +class HostHandler(Handler): + def __init__(self, level=0): + super().__init__(level=level) + + def emit(self, record: LogRecord): + host_log(record, "{}") + +oldBasicConfig = basicConfig +def basicConfig(*pargs, **kwargs): + if "handlers" not in kwargs: + kwargs["handlers"] = [HostHandler()] + return oldBasicConfig(*pargs, **kwargs) +"#, + target + ))?; + + let logging_scope = logging.dict(); + py.run(&code, Some(&logging_scope), None)?; + + let all = logging.index()?; + all.append("HostHandler")?; + + let logger = { + let get_logger = logging_scope.get_item("getLogger")?; + get_logger.call((target,), None)? + }; + + { + let basic_config = logging_scope.get_item("basicConfig")?; + basic_config.call( + (), + { + let dict = PyDict::new(py); + + // Ensure that all events are logged by setting + // the log level to NOTSET (we filter on rust's side) + dict.set_item("level", 0)?; + + Some(dict) + } + .as_ref(), + )?; + } + + { + let add_filter = logger.getattr("addFilter")?; + add_filter.call((wrap_pyfunction!(filter_error_log, &logging)?,), None)?; + } + + Ok(logger) +} diff --git a/crates/bytes/update.sh b/crates/yt_dlp/crates/pyo3-pylogger/update.sh index c1a0215..dd3e57e 100755 --- a/crates/bytes/update.sh +++ b/crates/yt_dlp/crates/pyo3-pylogger/update.sh @@ -1,9 +1,9 @@ -#!/usr/bin/env sh +#! /usr/bin/env sh # yt - A fully featured command line YouTube client # -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright (C) 2025 Dylan Bobby Storey <dylan.storey@gmail.com>, cpu <daniel@binaryparadox.net>, Warren Snipes <contact@warrensnipes.dev> +# SPDX-License-Identifier: Apache-2.0 # # This file is part of Yt. # @@ -13,3 +13,5 @@ cd "$(dirname "$0")" || exit 1 [ "$1" = "upgrade" ] && cargo upgrade --incompatible cargo update + +# vim: ft=sh diff --git a/crates/yt_dlp/examples/main.rs b/crates/yt_dlp/examples/main.rs new file mode 100644 index 0000000..e924407 --- /dev/null +++ b/crates/yt_dlp/examples/main.rs @@ -0,0 +1,15 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +fn main() { + let yt_dlp = yt_dlp::options::YoutubeDLOptions::new().build().unwrap(); + + dbg!(yt_dlp.version().unwrap()); +} diff --git a/crates/yt_dlp/src/duration.rs b/crates/yt_dlp/src/duration.rs deleted file mode 100644 index 19181a5..0000000 --- a/crates/yt_dlp/src/duration.rs +++ /dev/null @@ -1,78 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// TODO: This file should be de-duplicated with the same file in the 'yt' crate <2024-06-25> - -#[derive(Debug, Clone, Copy)] -pub struct Duration { - time: u32, -} - -impl From<&str> for Duration { - fn from(v: &str) -> Self { - let buf: Vec<_> = v.split(':').take(2).collect(); - Self { - time: (buf[0].parse::<u32>().expect("Should be a number") * 60) - + buf[1].parse::<u32>().expect("Should be a number"), - } - } -} - -impl From<Option<f64>> for Duration { - fn from(value: Option<f64>) -> Self { - Self { - #[allow( - clippy::cast_possible_truncation, - clippy::cast_precision_loss, - clippy::cast_sign_loss - )] - time: value.unwrap_or(0.0).ceil() as u32, - } - } -} - -impl std::fmt::Display for Duration { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - const SECOND: u32 = 1; - const MINUTE: u32 = 60 * SECOND; - const HOUR: u32 = 60 * MINUTE; - - let base_hour = self.time - (self.time % HOUR); - let base_min = (self.time % HOUR) - ((self.time % HOUR) % MINUTE); - let base_sec = (self.time % HOUR) % MINUTE; - - let h = base_hour / HOUR; - let m = base_min / MINUTE; - let s = base_sec / SECOND; - - if self.time == 0 { - write!(f, "0s") - } else if h > 0 { - write!(f, "{h}h {m}m") - } else { - write!(f, "{m}m {s}s") - } - } -} -#[cfg(test)] -mod test { - use super::Duration; - - #[test] - fn test_display_duration_1h() { - let dur = Duration { time: 60 * 60 }; - assert_eq!("1h 0m".to_owned(), dur.to_string()); - } - #[test] - fn test_display_duration_30min() { - let dur = Duration { time: 60 * 30 }; - assert_eq!("30m 0s".to_owned(), dur.to_string()); - } -} diff --git a/crates/yt_dlp/src/info_json.rs b/crates/yt_dlp/src/info_json.rs new file mode 100644 index 0000000..3ed08ee --- /dev/null +++ b/crates/yt_dlp/src/info_json.rs @@ -0,0 +1,56 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use pyo3::{ + Bound, Python, intern, + types::{PyAnyMethods, PyDict}, +}; + +pub type InfoJson = serde_json::Map<String, serde_json::Value>; + +/// # Panics +/// If expectation about python operations fail. +#[must_use] +pub fn json_loads( + input: serde_json::Map<String, serde_json::Value>, + py: Python<'_>, +) -> Bound<'_, PyDict> { + let json = py.import(intern!(py, "json")).expect("Module exists"); + let loads = json.getattr(intern!(py, "loads")).expect("Method exists"); + let self_str = serde_json::to_string(&serde_json::Value::Object(input)).expect("Vaild json"); + let dict = loads + .call((self_str,), None) + .expect("Vaild json is always a valid dict"); + + dict.downcast_into().expect("Should always be a dict") +} + +/// # Panics +/// If expectation about python operations fail. +#[must_use] +pub fn json_dumps(input: &Bound<'_, PyDict>) -> serde_json::Map<String, serde_json::Value> { + let py = input.py(); + + let json = py.import(intern!(py, "json")).expect("Module exists"); + let dumps = json.getattr(intern!(py, "dumps")).expect("Method exists"); + let dict = dumps + .call((input,), None) + .map_err(|err| err.print(py)) + .expect("Might not always work, but for our dicts it works"); + + let string: String = dict.extract().expect("Should always be a string"); + + let value: serde_json::Value = serde_json::from_str(&string).expect("Should be valid json"); + + match value { + serde_json::Value::Object(map) => map, + _ => unreachable!("These should not be json.dumps output"), + } +} diff --git a/crates/yt_dlp/src/lib.rs b/crates/yt_dlp/src/lib.rs index 970bfe2..6be5e87 100644 --- a/crates/yt_dlp/src/lib.rs +++ b/crates/yt_dlp/src/lib.rs @@ -1,6 +1,6 @@ // yt - A fully featured command line YouTube client // -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> // SPDX-License-Identifier: GPL-3.0-or-later // // This file is part of Yt. @@ -8,461 +8,371 @@ // You should have received a copy of the License along with this program. // If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -// The pyo3 `pyfunction` proc-macros call unsafe functions internally, which trigger this lint. -#![allow(unsafe_op_in_unsafe_fn)] -#![allow(clippy::missing_errors_doc)] +//! The `yt_dlp` interface is completely contained in the [`YoutubeDL`] structure. -use std::env; -use std::io::stdout; -use std::{fs::File, io::Write}; +use std::path::PathBuf; -use std::{path::PathBuf, sync::Once}; - -use crate::{duration::Duration, logging::setup_logging, wrapper::info_json::InfoJson}; - -use bytes::Bytes; -use log::{info, log_enabled, Level}; -use pyo3::types::{PyString, PyTuple, PyTupleMethods}; +use log::{debug, info}; use pyo3::{ - pyfunction, - types::{PyAnyMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyModule}, - wrap_pyfunction, Bound, PyAny, PyResult, Python, + Bound, Py, PyAny, Python, intern, + types::{PyAnyMethods, PyDict, PyIterator, PyList}, }; -use serde::Serialize; -use serde_json::{Map, Value}; use url::Url; -pub mod duration; -pub mod logging; -pub mod wrapper; - -#[cfg(test)] -mod tests; - -/// Synchronisation helper, to ensure that we don't setup the logger multiple times -static SYNC_OBJ: Once = Once::new(); - -/// Add a logger to the yt-dlp options. -/// If you have an logger set (i.e. for rust), than this will log to rust -/// -/// # Panics -/// This should never panic. -pub fn add_logger_and_sig_handler<'a>( - opts: Bound<'a, PyDict>, - py: Python<'_>, -) -> PyResult<Bound<'a, PyDict>> { - setup_logging(py, "yt_dlp")?; - - let logging = PyModule::import(py, "logging")?; - let ytdl_logger = logging.call_method1("getLogger", ("yt_dlp",))?; - - // Ensure that all events are logged by setting the log level to NOTSET (we filter on rust's side) - // Also use this static, to ensure that we don't configure the logger every time - SYNC_OBJ.call_once(|| { - // Disable the SIGINT (Ctrl+C) handler, python installs. - // This allows the user to actually stop the application with Ctrl+C. - // This is here because it can only be run in the main thread and this was here already. - py.run( - c"\ -import signal -signal.signal(signal.SIGINT, signal.SIG_DFL)", - None, - None, - ) - .expect("This code should always work"); - - let config_opts = PyDict::new(py); - config_opts - .set_item("level", 0) - .expect("Setting this item should always work"); - - logging - .call_method("basicConfig", (), Some(&config_opts)) - .expect("This method exists"); - }); - - // This was taken from `ytcc`, I don't think it is still applicable - // ytdl_logger.setattr("propagate", false)?; - // let logging_null_handler = logging.call_method0("NullHandler")?; - // ytdl_logger.setattr("addHandler", logging_null_handler)?; - - opts.set_item("logger", ytdl_logger).expect("Should work"); - - Ok(opts) -} +use crate::{ + info_json::{InfoJson, json_dumps, json_loads}, + python_error::{IntoPythonError, PythonError}, +}; -#[pyfunction] -#[allow(clippy::too_many_lines)] -#[allow(clippy::missing_panics_doc)] -#[allow(clippy::items_after_statements)] -#[allow( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::cast_precision_loss -)] -pub fn progress_hook(py: Python<'_>, input: &Bound<'_, PyDict>) -> PyResult<()> { - // Only add the handler, if the log-level is higher than Debug (this avoids covering debug - // messages). - if log_enabled!(Level::Debug) { - return Ok(()); - } +pub mod info_json; +pub mod options; +pub mod post_processors; +pub mod progress_hook; +pub mod python_error; + +#[macro_export] +macro_rules! json_get { + ($value:expr, $name:literal, $into:ident) => {{ + match $value.get($name) { + Some(val) => $crate::json_cast!(@log_key $name, val, $into), + None => panic!( + concat!( + "Expected '", + $name, + "' to be a key for the '", + stringify!($value), + "' object: {:#?}" + ), + $value + ), + } + }}; +} - // ANSI ESCAPE CODES Wrappers {{{ - // see: https://en.wikipedia.org/wiki/ANSI_escape_code#Control_Sequence_Introducer_commands - const CSI: &str = "\x1b["; - fn clear_whole_line() { - print!("{CSI}2K"); - } - fn move_to_col(x: usize) { - print!("{CSI}{x}G"); - } - // }}} - - let input: Map<String, Value> = serde_json::from_str(&json_dumps( - py, - input - .downcast::<PyAny>() - .expect("Will always work") - .to_owned(), - )?) - .expect("Python should always produce valid json"); - - macro_rules! get { - (@interrogate $item:ident, $type_fun:ident, $get_fun:ident, $name:expr) => {{ - let a = $item.get($name).expect(concat!( - "The field '", - stringify!($name), - "' should exist." - )); - - if a.$type_fun() { - a.$get_fun().expect( - "The should have been checked in the if guard, so unpacking here is fine", - ) +#[macro_export] +macro_rules! json_try_get { + ($value:expr, $name:literal, $into:ident) => {{ + if let Some(val) = $value.get($name) { + if val.is_null() { + None } else { - panic!( - "Value {} => \n{}\n is not of type: {}", - $name, - a, - stringify!($type_fun) - ); + Some(json_cast!(@log_key $name, val, $into)) } - }}; + } else { + None + } + }}; +} - ($type_fun:ident, $get_fun:ident, $name1:expr, $name2:expr) => {{ - let a = get! {@interrogate input, is_object, as_object, $name1}; - let b = get! {@interrogate a, $type_fun, $get_fun, $name2}; - b - }}; +#[macro_export] +macro_rules! json_cast { + ($value:expr, $into:ident) => {{ + let value_name = stringify!($value); + json_cast!(@log_key value_name, $value, $into) + }}; + + (@log_key $name:expr, $value:expr, $into:ident) => {{ + match $value.$into() { + Some(result) => result, + None => panic!( + concat!( + "Expected to be able to cast '{}' value (which is '{:?}') ", + stringify!($into) + ), + $name, + $value + ), + } + }}; +} - ($type_fun:ident, $get_fun:ident, $name:expr) => {{ - get! {@interrogate input, $type_fun, $get_fun, $name} - }}; - } +macro_rules! py_kw_args { + ($py:expr => $($kw_arg_name:ident = $kw_arg_val:expr),*) => {{ + use $crate::python_error::IntoPythonError; - macro_rules! default_get { - (@interrogate $item:ident, $default:expr, $get_fun:ident, $name:expr) => {{ - let a = if let Some(field) = $item.get($name) { - field.$get_fun().unwrap_or($default) - } else { - $default - }; - a - }}; - - ($get_fun:ident, $default:expr, $name1:expr, $name2:expr) => {{ - let a = get! {@interrogate input, is_object, as_object, $name1}; - let b = default_get! {@interrogate a, $default, $get_fun, $name2}; - b - }}; - - ($get_fun:ident, $default:expr, $name:expr) => {{ - default_get! {@interrogate input, $default, $get_fun, $name} - }}; - } + let dict = PyDict::new($py); - macro_rules! c { - ($color:expr, $format:expr) => { - format!("\x1b[{}m{}\x1b[0m", $color, $format) - }; - } + $( + dict.set_item(stringify!($kw_arg_name), $kw_arg_val).wrap_exc($py)?; + )* - fn format_bytes(bytes: u64) -> String { - let bytes = Bytes::new(bytes); - bytes.to_string() + Some(dict) } + .as_ref()}; +} +pub(crate) use py_kw_args; - fn format_speed(speed: f64) -> String { - #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] - let bytes = Bytes::new(speed.floor() as u64); - format!("{bytes}/s") +/// The core of the `yt_dlp` interface. +#[derive(Debug)] +pub struct YoutubeDL { + inner: Py<PyAny>, + options: serde_json::Map<String, serde_json::Value>, +} + +impl YoutubeDL { + /// Fetch the underlying `yt_dlp` and `python` version. + /// + /// # Errors + /// If python attribute access fails. + pub fn version(&self) -> Result<(String, String), PythonError> { + Python::with_gil(|py| { + let yt_dlp = py + .import(intern!(py, "yt_dlp")) + .wrap_exc(py)? + .getattr(intern!(py, "version")) + .wrap_exc(py)? + .getattr(intern!(py, "__version__")) + .wrap_exc(py)? + .extract() + .wrap_exc(py)?; + + let python = py.version(); + + Ok((yt_dlp, python.to_owned())) + }) } - let get_title = |add_extension: bool| -> String { - match get! {is_string, as_str, "info_dict", "ext"} { - "vtt" => { - format!( - "Subtitles ({})", - default_get! {as_str, "<No Subtitle Language>", "info_dict", "name"} - ) - } - title_extension @ ("webm" | "mp4" | "m4a") => { - if add_extension { - format!( - "{} ({})", - default_get! { as_str, "<No title>", "info_dict", "title"}, - title_extension - ) - } else { - default_get! { as_str, "<No title>", "info_dict", "title"}.to_owned() - } - } - other => panic!("The extension '{other}' is not yet implemented"), - } - }; - - match get! {is_string, as_str, "status"} { - "downloading" => { - let elapsed = default_get! {as_f64, 0.0f64, "elapsed"}; - let eta = default_get! {as_f64, 0.0, "eta"}; - let speed = default_get! {as_f64, 0.0, "speed"}; - - let downloaded_bytes = get! {is_u64, as_u64, "downloaded_bytes"}; - let (total_bytes, bytes_is_estimate): (u64, &'static str) = { - let total_bytes = default_get!(as_u64, 0, "total_bytes"); - if total_bytes == 0 { - let maybe_estimate = default_get!(as_u64, 0, "total_bytes_estimate"); - - if maybe_estimate == 0 { - // The download speed should be in bytes per second and the eta in seconds. - // Thus multiplying them gets us the raw bytes (which were estimated by `yt_dlp`, from their `info.json`) - let bytes_still_needed = (speed * eta).ceil() as u64; - - (downloaded_bytes + bytes_still_needed, "~") - } else { - (maybe_estimate, "~") - } - } else { - (total_bytes, "") - } - }; - let percent: f64 = { - if total_bytes == 0 { - 100.0 - } else { - (downloaded_bytes as f64 / total_bytes as f64) * 100.0 - } + /// Download a given list of URLs. + /// Returns the paths they were downloaded to. + /// + /// # Errors + /// If one of the downloads error. + pub fn download(&self, urls: &[Url]) -> Result<Vec<PathBuf>, extract_info::Error> { + let mut out_paths = Vec::with_capacity(urls.len()); + + for url in urls { + info!("Started downloading url: '{url}'"); + let info_json = self.extract_info(url, true, true)?; + + // Try to work around yt-dlp type weirdness + let result_string = if let Some(filename) = json_try_get!(info_json, "filename", as_str) + { + PathBuf::from(filename) + } else { + PathBuf::from(json_get!( + json_cast!( + json_get!(info_json, "requested_downloads", as_array)[0], + as_object + ), + "filename", + as_str + )) }; - clear_whole_line(); - move_to_col(1); - - print!( - "'{}' [{}/{} at {}] -> [{} of {}{} {}] ", - c!("34;1", get_title(true)), - c!("33;1", Duration::from(Some(elapsed))), - c!("33;1", Duration::from(Some(eta))), - c!("32;1", format_speed(speed)), - c!("31;1", format_bytes(downloaded_bytes)), - c!("31;1", bytes_is_estimate), - c!("31;1", format_bytes(total_bytes)), - c!("36;1", format!("{:.02}%", percent)) - ); - stdout().flush()?; - } - "finished" => { - println!("-> Finished downloading."); + out_paths.push(result_string); + info!("Finished downloading url"); } - "error" => { - panic!("-> Error while downloading: {}", get_title(true)) - } - other => unreachable!("'{other}' should not be a valid state!"), - }; - Ok(()) -} - -pub fn add_hooks<'a>(opts: Bound<'a, PyDict>, py: Python<'_>) -> PyResult<Bound<'a, PyDict>> { - if let Some(hooks) = opts.get_item("progress_hooks")? { - let hooks = hooks.downcast::<PyList>()?; - hooks.append(wrap_pyfunction!(progress_hook, py)?)?; - - opts.set_item("progress_hooks", hooks)?; - } else { - // No hooks are set yet - let hooks_list = PyList::new(py, &[wrap_pyfunction!(progress_hook, py)?])?; - - opts.set_item("progress_hooks", hooks_list)?; + Ok(out_paths) } - Ok(opts) -} - -/// `extract_info(self, url, download=True, ie_key=None, extra_info=None, process=True, force_generic_extractor=False)` -/// -/// Extract and return the information dictionary of the URL -/// -/// Arguments: -/// @param url URL to extract -/// -/// Keyword arguments: -/// @param download Whether to download videos -/// @param process Whether to resolve all unresolved references (URLs, playlist items). -/// Must be True for download to work -/// @param `ie_key` Use only the extractor with this key -/// -/// @param `extra_info` Dictionary containing the extra values to add to the info (For internal use only) -/// @`force_generic_extractor` Force using the generic extractor (Deprecated; use `ie_key`='Generic') -#[allow(clippy::unused_async)] -#[allow(clippy::missing_panics_doc)] -pub async fn extract_info( - yt_dlp_opts: &Map<String, Value>, - url: &Url, - download: bool, - process: bool, -) -> PyResult<InfoJson> { - Python::with_gil(|py| { - let opts = json_map_to_py_dict(yt_dlp_opts, py)?; - - let instance = get_yt_dlp(py, opts)?; - let args = (url.as_str(),); - - let kwargs = PyDict::new(py); - kwargs.set_item("download", download)?; - kwargs.set_item("process", process)?; - - let result = instance.call_method("extract_info", args, Some(&kwargs))?; - - // Remove the `<generator at 0xsome_hex>`, by setting it to null - if !process { - result.set_item("entries", ())?; - } + /// `extract_info(self, url, download=True, ie_key=None, extra_info=None, process=True, force_generic_extractor=False)` + /// + /// Extract and return the information dictionary of the URL + /// + /// Arguments: + /// - `url` URL to extract + /// + /// Keyword arguments: + /// :`download` Whether to download videos + /// :`process` Whether to resolve all unresolved references (URLs, playlist items). + /// Must be True for download to work + /// + /// # Panics + /// If expectations about python fail to hold. + /// + /// # Errors + /// If python operations fail. + pub fn extract_info( + &self, + url: &Url, + download: bool, + process: bool, + ) -> Result<InfoJson, extract_info::Error> { + Python::with_gil(|py| { + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "extract_info")) + .wrap_exc(py)?; + + let result = inner + .call( + (url.to_string(),), + py_kw_args!(py => download = download, process = process), + ) + .wrap_exc(py)? + .downcast_into::<PyDict>() + .expect("This is a dict"); + + // Resolve the generator object + if let Ok(generator) = result.get_item(intern!(py, "entries")) { + if generator.is_instance_of::<PyList>() { + // already resolved. Do nothing + } else if let Ok(generator) = generator.downcast::<PyIterator>() { + // A python generator object. + let max_backlog = json_try_get!(self.options, "playlistend", as_u64) + .map_or(10000, |playlistend| { + usize::try_from(playlistend).expect("Should work") + }); + + let mut out = vec![]; + for output in generator { + out.push(output.wrap_exc(py)?); + + if out.len() == max_backlog { + break; + } + } - let result_str = json_dumps(py, result)?; + result.set_item(intern!(py, "entries"), out).wrap_exc(py)?; + } else { + // Probably some sort of paged list (`OnDemand` or otherwise) + let max_backlog = json_try_get!(self.options, "playlistend", as_u64) + .map_or(10000, |playlistend| { + usize::try_from(playlistend).expect("Should work") + }); - if let Ok(confirm) = env::var("YT_STORE_INFO_JSON") { - if confirm == "yes" { - let mut file = File::create("output.info.json")?; - write!(file, "{result_str}").unwrap(); - } - } + let next = generator.getattr(intern!(py, "getslice")).wrap_exc(py)?; - Ok(serde_json::from_str(&result_str) - .expect("Python should be able to produce correct json")) - }) -} + let output = next + .call((), py_kw_args!(py => start = 0, end = max_backlog)) + .wrap_exc(py)?; -/// # Panics -/// Only if python fails to return a valid URL. -pub fn unsmuggle_url(smug_url: &Url) -> PyResult<Url> { - Python::with_gil(|py| { - let utils = get_yt_dlp_utils(py)?; - let url = utils - .call_method1("unsmuggle_url", (smug_url.as_str(),))? - .downcast::<PyTuple>()? - .get_item(0)?; - - let url: Url = url - .downcast::<PyString>()? - .to_string() - .parse() - .expect("Python should be able to return a valid url"); - - Ok(url) - }) -} + result + .set_item(intern!(py, "entries"), output) + .wrap_exc(py)?; + } + } -/// Download a given list of URLs. -/// Returns the paths they were downloaded to. -/// -/// # Panics -/// Only if `yt_dlp` changes their `info_json` schema. -pub async fn download( - urls: &[Url], - download_options: &Map<String, Value>, -) -> PyResult<Vec<PathBuf>> { - let mut out_paths = Vec::with_capacity(urls.len()); - - for url in urls { - info!("Started downloading url: '{}'", url); - let info_json = extract_info(download_options, url, true, true).await?; - - // Try to work around yt-dlp type weirdness - let result_string = if let Some(filename) = info_json.filename { - filename - } else { - info_json.requested_downloads.expect("This must exist")[0] - .filename - .clone() - }; + let result = self.prepare_info_json(&result, py)?; - out_paths.push(result_string); - info!("Finished downloading url: '{}'", url); + Ok(result) + }) } - Ok(out_paths) -} - -fn json_map_to_py_dict<'a>( - map: &Map<String, Value>, - py: Python<'a>, -) -> PyResult<Bound<'a, PyDict>> { - let json_string = serde_json::to_string(&map).expect("This must always work"); + /// Take the (potentially modified) result of the information extractor (i.e., + /// [`Self::extract_info`] with `process` and `download` set to false) + /// and resolve all unresolved references (URLs, + /// playlist items). + /// + /// It will also download the videos if 'download' is true. + /// Returns the resolved `ie_result`. + /// + /// # Panics + /// If expectations about python fail to hold. + /// + /// # Errors + /// If python operations fail. + pub fn process_ie_result( + &self, + ie_result: InfoJson, + download: bool, + ) -> Result<InfoJson, process_ie_result::Error> { + Python::with_gil(|py| { + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "process_ie_result")) + .wrap_exc(py)?; + + let result = inner + .call( + (json_loads(ie_result, py),), + py_kw_args!(py => download = download), + ) + .wrap_exc(py)? + .downcast_into::<PyDict>() + .expect("This is a dict"); - let python_dict = json_loads(py, json_string)?; + let result = self.prepare_info_json(&result, py)?; - Ok(python_dict) -} + Ok(result) + }) + } -fn json_dumps(py: Python<'_>, input: Bound<'_, PyAny>) -> PyResult<String> { - // json.dumps(yt_dlp.sanitize_info(input)) + /// Close this [`YoutubeDL`] instance, and stop all currently running downloads. + /// + /// # Errors + /// If python operations fail. + pub fn close(&self) -> Result<(), close::Error> { + Python::with_gil(|py| { + debug!("Closing YoutubeDL."); - let yt_dlp = get_yt_dlp(py, PyDict::new(py))?; - let sanitized_result = yt_dlp.call_method1("sanitize_info", (input,))?; + let inner = self + .inner + .bind(py) + .getattr(intern!(py, "close")) + .wrap_exc(py)?; - let json = PyModule::import(py, "json")?; - let dumps = json.getattr("dumps")?; + inner.call0().wrap_exc(py)?; - let output = dumps.call1((sanitized_result,))?; + Ok(()) + }) + } - let output_str = output.extract::<String>()?; + fn prepare_info_json<'py>( + &self, + info: &Bound<'py, PyDict>, + py: Python<'py>, + ) -> Result<InfoJson, prepare::Error> { + let sanitize = self + .inner + .bind(py) + .getattr(intern!(py, "sanitize_info")) + .wrap_exc(py)?; - Ok(output_str) -} + let value = sanitize.call((info,), None).wrap_exc(py)?; -fn json_loads_str<T: Serialize>(py: Python<'_>, input: T) -> PyResult<Bound<'_, PyDict>> { - let string = serde_json::to_string(&input).expect("Correct json must be pased"); + let result = value.downcast::<PyDict>().expect("This should stay a dict"); - json_loads(py, string) + Ok(json_dumps(result)) + } } -fn json_loads(py: Python<'_>, input: String) -> PyResult<Bound<'_, PyDict>> { - // json.loads(input) - - let json = PyModule::import(py, "json")?; - let dumps = json.getattr("loads")?; +#[allow(missing_docs)] +pub mod close { + use crate::python_error::PythonError; - let output = dumps.call1((input,))?; - - Ok(output - .downcast::<PyDict>() - .expect("This should always be a PyDict") - .clone()) + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + } } +#[allow(missing_docs)] +pub mod process_ie_result { + use crate::{prepare, python_error::PythonError}; -fn get_yt_dlp_utils(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> { - let yt_dlp = PyModule::import(py, "yt_dlp")?; - let utils = yt_dlp.getattr("utils")?; + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), - Ok(utils) + #[error("Failed to prepare the info json")] + InfoJsonPrepare(#[from] prepare::Error), + } } -fn get_yt_dlp<'a>(py: Python<'a>, opts: Bound<'a, PyDict>) -> PyResult<Bound<'a, PyAny>> { - // Unconditionally set a logger - let opts = add_logger_and_sig_handler(opts, py)?; - let opts = add_hooks(opts, py)?; +#[allow(missing_docs)] +pub mod extract_info { + use crate::{prepare, python_error::PythonError}; - let yt_dlp = PyModule::import(py, "yt_dlp")?; - let youtube_dl = yt_dlp.call_method1("YoutubeDL", (opts,))?; + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), - Ok(youtube_dl) + #[error("Failed to prepare the info json")] + InfoJsonPrepare(#[from] prepare::Error), + } +} +#[allow(missing_docs)] +pub mod prepare { + use crate::python_error::PythonError; + + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + } } diff --git a/crates/yt_dlp/src/logging.rs b/crates/yt_dlp/src/logging.rs deleted file mode 100644 index 670fc1c..0000000 --- a/crates/yt_dlp/src/logging.rs +++ /dev/null @@ -1,132 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// This file is taken from: https://github.com/dylanbstorey/pyo3-pylogger/blob/d89e0d6820ebc4f067647e3b74af59dbc4941dd5/src/lib.rs -// It is licensed under the Apache 2.0 License, copyright up to 2024 by Dylan Storey -// It was modified by Benedikt Peetz 2024 - -// The pyo3 `pyfunction` proc-macros call unsafe functions internally, which trigger this lint. -#![allow(unsafe_op_in_unsafe_fn)] - -use std::ffi::CString; - -use log::{logger, Level, MetadataBuilder, Record}; -use pyo3::{ - prelude::{PyAnyMethods, PyListMethods, PyModuleMethods}, - pyfunction, wrap_pyfunction, Bound, PyAny, PyResult, Python, -}; - -/// Consume a Python `logging.LogRecord` and emit a Rust `Log` instead. -#[allow(clippy::needless_pass_by_value)] -#[pyfunction] -fn host_log(record: Bound<'_, PyAny>, rust_target: &str) -> PyResult<()> { - let level = record.getattr("levelno")?; - let message = record.getattr("getMessage")?.call0()?.to_string(); - let pathname = record.getattr("pathname")?.to_string(); - let lineno = record - .getattr("lineno")? - .to_string() - .parse::<u32>() - .expect("This should always be a u32"); - - let logger_name = record.getattr("name")?.to_string(); - - let full_target: Option<String> = if logger_name.trim().is_empty() || logger_name == "root" { - None - } else { - // Libraries (ex: tracing_subscriber::filter::Directive) expect rust-style targets like foo::bar, - // and may not deal well with "." as a module separator: - let logger_name = logger_name.replace('.', "::"); - Some(format!("{rust_target}::{logger_name}")) - }; - - let target = full_target.as_deref().unwrap_or(rust_target); - - // error - let error_metadata = if level.ge(40u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Error) - .build() - } else if level.ge(30u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Warn) - .build() - } else if level.ge(20u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Info) - .build() - } else if level.ge(10u8)? { - MetadataBuilder::new() - .target(target) - .level(Level::Debug) - .build() - } else { - MetadataBuilder::new() - .target(target) - .level(Level::Trace) - .build() - }; - - logger().log( - &Record::builder() - .metadata(error_metadata) - .args(format_args!("{}", &message)) - .line(Some(lineno)) - .file(None) - .module_path(Some(&pathname)) - .build(), - ); - - Ok(()) -} - -/// Registers the `host_log` function in rust as the event handler for Python's logging logger -/// This function needs to be called from within a pyo3 context as early as possible to ensure logging messages -/// arrive to the rust consumer. -/// -/// # Panics -/// Only if internal assertions fail. -#[allow(clippy::module_name_repetitions)] -pub fn setup_logging(py: Python<'_>, target: &str) -> PyResult<()> { - let logging = py.import("logging")?; - - logging.setattr("host_log", wrap_pyfunction!(host_log, &logging)?)?; - - py.run( - CString::new(format!( - r#" -class HostHandler(Handler): - def __init__(self, level=0): - super().__init__(level=level) - - def emit(self, record): - host_log(record,"{target}") - -oldBasicConfig = basicConfig -def basicConfig(*pargs, **kwargs): - if "handlers" not in kwargs: - kwargs["handlers"] = [HostHandler()] - return oldBasicConfig(*pargs, **kwargs) -"# - )) - .expect("This is hardcoded") - .as_c_str(), - Some(&logging.dict()), - None, - )?; - - let all = logging.index()?; - all.append("HostHandler")?; - - Ok(()) -} diff --git a/crates/yt_dlp/src/options.rs b/crates/yt_dlp/src/options.rs new file mode 100644 index 0000000..ad30301 --- /dev/null +++ b/crates/yt_dlp/src/options.rs @@ -0,0 +1,207 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::sync; + +use pyo3::{ + Bound, IntoPyObjectExt, PyAny, PyResult, Python, intern, + types::{PyAnyMethods, PyCFunction, PyDict, PyTuple}, +}; +use pyo3_pylogger::setup_logging; + +use crate::{ + YoutubeDL, json_loads, post_processors, py_kw_args, + python_error::{IntoPythonError, PythonError}, +}; + +pub type ProgressHookFunction = fn(py: Python<'_>) -> PyResult<Bound<'_, PyCFunction>>; +pub type PostProcessorFunction = fn(py: Python<'_>) -> PyResult<Bound<'_, PyAny>>; + +/// Options, that are used to customize the download behaviour. +/// +/// In the future, this might get a Builder api. +/// +/// See `help(yt_dlp.YoutubeDL())` from python for a full list of available options. +#[derive(Default, Debug)] +pub struct YoutubeDLOptions { + options: serde_json::Map<String, serde_json::Value>, + progress_hook: Option<ProgressHookFunction>, + post_processors: Vec<PostProcessorFunction>, +} + +impl YoutubeDLOptions { + #[must_use] + pub fn new() -> Self { + let me = Self { + options: serde_json::Map::new(), + progress_hook: None, + post_processors: vec![], + }; + + me.with_post_processor(post_processors::dearrow::process) + } + + #[must_use] + pub fn set(self, key: impl Into<String>, value: impl Into<serde_json::Value>) -> Self { + let mut options = self.options; + options.insert(key.into(), value.into()); + + Self { options, ..self } + } + + #[must_use] + pub fn with_progress_hook(self, progress_hook: ProgressHookFunction) -> Self { + if let Some(_previous_hook) = self.progress_hook { + todo!() + } else { + Self { + progress_hook: Some(progress_hook), + ..self + } + } + } + + #[must_use] + pub fn with_post_processor(mut self, pp: PostProcessorFunction) -> Self { + self.post_processors.push(pp); + self + } + + /// # Errors + /// If the underlying [`YoutubeDL::from_options`] errors. + pub fn build(self) -> Result<YoutubeDL, build::Error> { + YoutubeDL::from_options(self) + } + + #[must_use] + pub fn from_json_options(options: serde_json::Map<String, serde_json::Value>) -> Self { + Self { + options, + ..Self::new() + } + } + + #[must_use] + pub fn get(&self, key: &str) -> Option<&serde_json::Value> { + self.options.get(key) + } +} + +impl YoutubeDL { + /// Construct this instance from options. + /// + /// # Panics + /// If `yt_dlp` changed their interface. + /// + /// # Errors + /// If a python call fails. + #[allow(clippy::too_many_lines)] + pub fn from_options(options: YoutubeDLOptions) -> Result<Self, build::Error> { + pyo3::prepare_freethreaded_python(); + + let output_options = options.options.clone(); + + let yt_dlp_module = Python::with_gil(|py| { + let opts = json_loads(options.options, py); + + { + static CALL_ONCE: sync::Once = sync::Once::new(); + + CALL_ONCE.call_once(|| { + py.run( + c" +import signal +signal.signal(signal.SIGINT, signal.SIG_DFL) + ", + None, + None, + ) + .unwrap_or_else(|err| { + panic!("Failed to disable python signal handling: {err}") + }); + }); + } + + { + // Setup the progress hook + if let Some(ph) = options.progress_hook { + opts.set_item(intern!(py, "progress_hooks"), vec![ph(py).wrap_exc(py)?]) + .wrap_exc(py)?; + } + } + + { + // Unconditionally set a logger. + // Otherwise, yt_dlp will log to stderr. + + let ytdl_logger = setup_logging(py, "yt_dlp").wrap_exc(py)?; + + opts.set_item(intern!(py, "logger"), ytdl_logger) + .wrap_exc(py)?; + } + + let inner = { + let p_params = opts.into_bound_py_any(py).wrap_exc(py)?; + let p_auto_init = true.into_bound_py_any(py).wrap_exc(py)?; + + py.import(intern!(py, "yt_dlp.YoutubeDL")) + .wrap_exc(py)? + .getattr(intern!(py, "YoutubeDL")) + .wrap_exc(py)? + .call1( + PyTuple::new( + py, + [ + p_params.into_bound_py_any(py).wrap_exc(py)?, + p_auto_init.into_bound_py_any(py).wrap_exc(py)?, + ], + ) + .wrap_exc(py)?, + ) + .wrap_exc(py)? + }; + + { + // Setup the post processors + let add_post_processor_fun = inner + .getattr(intern!(py, "add_post_processor")) + .wrap_exc(py)?; + + for pp in options.post_processors { + add_post_processor_fun + .call( + (pp(py).wrap_exc(py)?.into_bound_py_any(py).wrap_exc(py)?,), + // "when" can take any value in yt_dlp.utils.POSTPROCESS_WHEN + py_kw_args!(py => when = "pre_process"), + ) + .wrap_exc(py)?; + } + } + + Ok::<_, PythonError>(inner.unbind()) + })?; + + Ok(Self { + inner: yt_dlp_module, + options: output_options, + }) + } +} + +#[allow(missing_docs)] +pub mod build { + use crate::python_error::PythonError; + + #[derive(Debug, thiserror::Error)] + pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + } +} diff --git a/crates/yt_dlp/src/post_processors/dearrow.rs b/crates/yt_dlp/src/post_processors/dearrow.rs new file mode 100644 index 0000000..f35f301 --- /dev/null +++ b/crates/yt_dlp/src/post_processors/dearrow.rs @@ -0,0 +1,247 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use curl::easy::Easy; +use log::{error, info, trace, warn}; +use pyo3::{ + Bound, PyAny, PyErr, PyResult, Python, exceptions, intern, pyfunction, + types::{PyAnyMethods, PyDict, PyModule}, + wrap_pyfunction, +}; +use serde::{Deserialize, Serialize}; + +use crate::{ + pydict_cast, pydict_get, + python_error::{IntoPythonError, PythonError}, +}; + +/// # Errors +/// - If the underlying function returns an error. +/// - If python operations fail. +pub fn process(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> { + #[pyfunction] + fn actual_processor(info_json: Bound<'_, PyDict>) -> PyResult<Bound<'_, PyDict>> { + let output = match unwrapped_process(info_json) { + Ok(ok) => ok, + Err(err) => { + return Err(PyErr::new::<exceptions::PyRuntimeError, _>(err.to_string())); + } + }; + Ok(output) + } + + let module = PyModule::new(py, "rust_post_processors")?; + let scope = PyDict::new(py); + scope.set_item( + intern!(py, "actual_processor"), + wrap_pyfunction!(actual_processor, module)?, + )?; + py.run( + c" +import yt_dlp + +class DeArrow(yt_dlp.postprocessor.PostProcessor): + def run(self, info): + info = actual_processor(info) + return [], info + +inst = DeArrow() +", + Some(&scope), + None, + )?; + + Ok(scope.get_item(intern!(py, "inst"))?.downcast_into()?) +} + +/// # Errors +/// If the API access fails. +pub fn unwrapped_process(info: Bound<'_, PyDict>) -> Result<Bound<'_, PyDict>, Error> { + if pydict_get!(info, "extractor_key", String).as_str() != "Youtube" { + return Ok(info); + } + + let mut retry_num = 3; + let mut output: DeArrowApi = { + loop { + let output_bytes = { + let mut dst = Vec::new(); + + let mut easy = Easy::new(); + easy.url( + format!( + "https://sponsor.ajay.app/api/branding?videoID={}", + pydict_get!(info, "id", String) + ) + .as_str(), + )?; + + let mut transfer = easy.transfer(); + transfer.write_function(|data| { + dst.extend_from_slice(data); + Ok(data.len()) + })?; + transfer.perform()?; + drop(transfer); + + dst + }; + + match serde_json::from_slice(&output_bytes) { + Ok(ok) => break ok, + Err(err) => { + if retry_num > 0 { + trace!( + "DeArrow: Api access failed, trying again ({retry_num} retries left)" + ); + retry_num -= 1; + } else { + let err: serde_json::Error = err; + return Err(err.into()); + } + } + } + } + }; + + // We pop the titles, so we need this vector reversed. + output.titles.reverse(); + + let title_len = output.titles.len(); + let mut iterator = output.titles.clone(); + let selected = loop { + let Some(title) = iterator.pop() else { + break false; + }; + + if (title.locked || title.votes < 1) && title_len > 1 { + info!( + "DeArrow: Skipping title {:#?}, as it is not good enough", + title.value + ); + // Skip titles that are not “good” enough. + continue; + } + + update_title(&info, &title.value).wrap_exc(info.py())?; + + break true; + }; + + if !selected && title_len != 0 { + // No title was selected, even though we had some titles. + // Just pick the first one in this case. + update_title(&info, &output.titles[0].value).wrap_exc(info.py())?; + } + + Ok(info) +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + Python(#[from] PythonError), + + #[error("Failed to access the DeArrow api: {0}")] + Get(#[from] curl::Error), + + #[error("Failed to deserialize a api json return object: {0}")] + Deserialize(#[from] serde_json::Error), +} + +fn update_title(info: &Bound<'_, PyDict>, new_title: &str) -> PyResult<()> { + let py = info.py(); + + assert!(!info.contains(intern!(py, "original_title"))?); + + if let Ok(old_title) = info.get_item(intern!(py, "title")) { + warn!( + "DeArrow: Updating title from {:#?} to {:#?}", + pydict_cast!(old_title, &str), + new_title + ); + + info.set_item(intern!(py, "original_title"), old_title) + .expect("We checked, it is a new key"); + } else { + warn!("DeArrow: Setting title to {new_title:#?}"); + } + + let cleaned_title = { + // NOTE(@bpeetz): DeArrow uses `>` as a “Don't format the next word” mark. + // They should be removed, if one does not use a auto-formatter. <2025-06-16> + new_title.replace('>', "") + }; + + info.set_item(intern!(py, "title"), cleaned_title) + .expect("This should work?"); + + Ok(()) +} + +#[derive(Serialize, Deserialize)] +/// See: <https://wiki.sponsor.ajay.app/w/API_Docs/DeArrow> +struct DeArrowApi { + titles: Vec<Title>, + thumbnails: Vec<Thumbnail>, + + #[serde(alias = "randomTime")] + random_time: Option<f64>, + + #[serde(alias = "videoDuration")] + video_duration: Option<f64>, + + #[serde(alias = "casualVotes")] + casual_votes: Vec<CasualVote>, +} + +#[derive(Serialize, Deserialize)] +struct CasualVote { + id: String, + count: u32, + title: String, +} + +#[derive(Serialize, Deserialize, Clone)] +struct Title { + /// Note: Titles will sometimes contain > before a word. + /// This tells the auto-formatter to not format a word. + /// If you have no auto-formatter, you can ignore this and replace it with an empty string + #[serde(alias = "title")] + value: String, + + original: bool, + votes: u64, + locked: bool, + + #[serde(alias = "UUID")] + uuid: String, + + /// only present if requested + #[serde(alias = "userID")] + user_id: Option<String>, +} + +#[derive(Serialize, Deserialize)] +struct Thumbnail { + // null if original is true + timestamp: Option<f64>, + + original: bool, + votes: u64, + locked: bool, + + #[serde(alias = "UUID")] + uuid: String, + + /// only present if requested + #[serde(alias = "userID")] + user_id: Option<String>, +} diff --git a/crates/yt_dlp/src/post_processors/mod.rs b/crates/yt_dlp/src/post_processors/mod.rs new file mode 100644 index 0000000..d9be3f5 --- /dev/null +++ b/crates/yt_dlp/src/post_processors/mod.rs @@ -0,0 +1,48 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub mod dearrow; + +#[macro_export] +macro_rules! pydict_get { + ($value:expr, $name:literal, $into:ty) => {{ + let item = $value.get_item(pyo3::intern!($value.py(), $name)); + match &item { + Ok(val) => $crate::pydict_cast!(val, $into), + Err(_) => panic!( + concat!( + "Expected '", + $name, + "' to be a key for the'", + stringify!($value), + "' py dictionary: {:#?}" + ), + $value + ), + } + }}; +} + +#[macro_export] +macro_rules! pydict_cast { + ($value:expr, $into:ty) => {{ + match $value.extract::<$into>() { + Ok(result) => result, + Err(val) => panic!( + concat!( + "Expected to be able to extract ", + stringify!($into), + " from value ({:#?})." + ), + val + ), + } + }}; +} diff --git a/crates/yt_dlp/src/progress_hook.rs b/crates/yt_dlp/src/progress_hook.rs new file mode 100644 index 0000000..7e5f8a5 --- /dev/null +++ b/crates/yt_dlp/src/progress_hook.rs @@ -0,0 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +#[macro_export] +macro_rules! wrap_progress_hook { + ($name:ident, $new_name:ident) => { + pub(crate) fn $new_name( + py: yt_dlp::progress_hook::__priv::pyo3::Python<'_>, + ) -> yt_dlp::progress_hook::__priv::pyo3::PyResult< + yt_dlp::progress_hook::__priv::pyo3::Bound< + '_, + yt_dlp::progress_hook::__priv::pyo3::types::PyCFunction, + >, + > { + #[yt_dlp::progress_hook::__priv::pyo3::pyfunction] + #[pyo3(crate = "yt_dlp::progress_hook::__priv::pyo3")] + fn inner( + input: yt_dlp::progress_hook::__priv::pyo3::Bound< + '_, + yt_dlp::progress_hook::__priv::pyo3::types::PyDict, + >, + ) -> yt_dlp::progress_hook::__priv::pyo3::PyResult<()> { + let processed_input = { + let new_dict = yt_dlp::progress_hook::__priv::pyo3::types::PyDict::new(input.py()); + + input + .into_iter() + .filter_map(|(name, value)| { + let real_name = yt_dlp::progress_hook::__priv::pyo3::types::PyAnyMethods::extract::<String>(&name).expect("Should always be a string"); + + if real_name.starts_with('_') { + None + } else { + Some((real_name, value)) + } + }) + .for_each(|(key, value)| { + yt_dlp::progress_hook::__priv::pyo3::types::PyDictMethods::set_item(&new_dict, &key, value) + .expect("This is a transpositions, should always be valid"); + }); + yt_dlp::progress_hook::__priv::json_dumps(&new_dict) + }; + + $name(processed_input)?; + + Ok(()) + } + + let module = yt_dlp::progress_hook::__priv::pyo3::types::PyModule::new(py, "progress_hook")?; + let fun = yt_dlp::progress_hook::__priv::pyo3::wrap_pyfunction!(inner, module)?; + + Ok(fun) + } + }; +} + +pub mod __priv { + pub use crate::info_json::{json_dumps, json_loads}; + pub use pyo3; +} diff --git a/crates/yt_dlp/src/python_error.rs b/crates/yt_dlp/src/python_error.rs new file mode 100644 index 0000000..0c442b3 --- /dev/null +++ b/crates/yt_dlp/src/python_error.rs @@ -0,0 +1,55 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::{self, Display}; + +use log::{Level, debug, log_enabled}; +use pyo3::{PyErr, Python, types::PyTracebackMethods}; + +#[derive(thiserror::Error, Debug)] +pub struct PythonError(pub String); + +pub(crate) trait IntoPythonError<T>: Sized { + fn wrap_exc(self, py: Python<'_>) -> Result<T, PythonError>; +} + +impl<T> IntoPythonError<T> for Result<T, PyErr> { + fn wrap_exc(self, py: Python<'_>) -> Result<T, PythonError> { + self.map_err(|exc| PythonError::from_exception(py, &exc)) + } +} + +impl Display for PythonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Python threw an exception: {}", self.0) + } +} + +impl PythonError { + pub(super) fn from_exception(py: Python<'_>, exc: &PyErr) -> Self { + let buffer = process_exception(py, exc); + Self(buffer) + } +} + +pub(super) fn process_exception(py: Python<'_>, err: &PyErr) -> String { + if log_enabled!(Level::Debug) { + let mut output = err.to_string(); + + if let Some(tb) = err.traceback(py) { + output.push('\n'); + output.push_str(&tb.format().unwrap()); + } + + debug!("Python threw an exception: {output}"); + } + + err.to_string() +} diff --git a/crates/yt_dlp/src/tests.rs b/crates/yt_dlp/src/tests.rs deleted file mode 100644 index b48deb4..0000000 --- a/crates/yt_dlp/src/tests.rs +++ /dev/null @@ -1,85 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use std::sync::LazyLock; - -use serde_json::{json, Value}; -use url::Url; - -static YT_OPTS: LazyLock<serde_json::Map<String, Value>> = LazyLock::new(|| { - match json!({ - "playliststart": 1, - "playlistend": 10, - "noplaylist": false, - "extract_flat": false, - }) { - Value::Object(obj) => obj, - _ => unreachable!("This json is hardcoded"), - } -}); - -#[tokio::test] -async fn test_extract_info_video() { - let info = crate::extract_info( - &YT_OPTS, - &Url::parse("https://www.youtube.com/watch?v=dbjPnXaacAU").expect("Is valid."), - false, - false, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{info:#?}"); -} - -#[tokio::test] -async fn test_extract_info_url() { - let err = crate::extract_info( - &YT_OPTS, - &Url::parse("https://google.com").expect("Is valid."), - false, - false, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{err:#?}"); -} - -#[tokio::test] -async fn test_extract_info_playlist() { - let err = crate::extract_info( - &YT_OPTS, - &Url::parse("https://www.youtube.com/@TheGarriFrischer/videos").expect("Is valid."), - false, - true, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{err:#?}"); -} -#[tokio::test] -async fn test_extract_info_playlist_full() { - let err = crate::extract_info( - &YT_OPTS, - &Url::parse("https://www.youtube.com/@NixOS-Foundation/videos").expect("Is valid."), - false, - true, - ) - .await - .map_err(|err| format!("Encountered error: '{err}'")) - .unwrap(); - - println!("{err:#?}"); -} diff --git a/crates/yt_dlp/src/wrapper/info_json.rs b/crates/yt_dlp/src/wrapper/info_json.rs deleted file mode 100644 index 35d155e..0000000 --- a/crates/yt_dlp/src/wrapper/info_json.rs +++ /dev/null @@ -1,556 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -// `yt_dlp` named them like this. -#![allow(clippy::pub_underscore_fields)] - -use std::{collections::HashMap, path::PathBuf}; - -use pyo3::{types::PyDict, Bound, PyResult, Python}; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::Value; -use url::Url; - -use crate::json_loads_str; - -type Todo = String; -type Extractor = String; -type ExtractorKey = String; - -// TODO: Change this to map `_type` to a structure of values, instead of the options <2024-05-27> -// And replace all the strings with better types (enums or urls) -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct InfoJson { - pub __files_to_move: Option<FilesToMove>, - pub __last_playlist_index: Option<u32>, - pub __post_extractor: Option<String>, - pub __x_forwarded_for_ip: Option<String>, - pub _filename: Option<PathBuf>, - pub _format_sort_fields: Option<Vec<String>>, - pub _has_drm: Option<Todo>, - pub _type: Option<InfoType>, - pub _version: Option<Version>, - pub abr: Option<f64>, - pub acodec: Option<String>, - pub age_limit: Option<u32>, - pub aspect_ratio: Option<f64>, - pub asr: Option<u32>, - pub audio_channels: Option<u32>, - pub audio_ext: Option<String>, - pub automatic_captions: Option<HashMap<String, Vec<Caption>>>, - pub availability: Option<String>, - pub average_rating: Option<String>, - pub categories: Option<Vec<String>>, - pub channel: Option<String>, - pub channel_follower_count: Option<u32>, - pub channel_id: Option<String>, - pub channel_is_verified: Option<bool>, - pub channel_url: Option<String>, - pub chapters: Option<Vec<Chapter>>, - pub comment_count: Option<u32>, - pub comments: Option<Vec<Comment>>, - pub concurrent_view_count: Option<u32>, - pub description: Option<String>, - pub display_id: Option<String>, - pub downloader_options: Option<DownloaderOptions>, - pub duration: Option<f64>, - pub duration_string: Option<String>, - pub dynamic_range: Option<String>, - pub entries: Option<Vec<InfoJson>>, - pub episode: Option<String>, - pub episode_number: Option<u32>, - pub epoch: Option<u32>, - pub ext: Option<String>, - pub extractor: Option<Extractor>, - pub extractor_key: Option<ExtractorKey>, - pub filename: Option<PathBuf>, - pub filesize: Option<u64>, - pub filesize_approx: Option<u64>, - pub format: Option<String>, - pub format_id: Option<String>, - pub format_index: Option<u32>, - pub format_note: Option<String>, - pub formats: Option<Vec<Format>>, - pub fps: Option<f64>, - pub fulltitle: Option<String>, - pub has_drm: Option<bool>, - pub heatmap: Option<Vec<HeatMapEntry>>, - pub height: Option<u32>, - pub http_headers: Option<HttpHeader>, - pub id: Option<String>, - pub ie_key: Option<ExtractorKey>, - pub is_live: Option<bool>, - pub language: Option<String>, - pub language_preference: Option<i32>, - pub license: Option<Todo>, - pub like_count: Option<u32>, - pub live_status: Option<String>, - pub location: Option<Todo>, - pub manifest_url: Option<Url>, - pub modified_date: Option<String>, - pub n_entries: Option<u32>, - pub original_url: Option<String>, - pub playable_in_embed: Option<bool>, - pub playlist: Option<Todo>, - pub playlist_autonumber: Option<u32>, - pub playlist_channel: Option<Todo>, - pub playlist_channel_id: Option<Todo>, - pub playlist_count: Option<u32>, - pub playlist_id: Option<Todo>, - pub playlist_index: Option<u64>, - pub playlist_title: Option<Todo>, - pub playlist_uploader: Option<Todo>, - pub playlist_uploader_id: Option<Todo>, - pub preference: Option<Todo>, - pub protocol: Option<String>, - pub quality: Option<f64>, - pub release_date: Option<String>, - pub release_timestamp: Option<u64>, - pub release_year: Option<u32>, - pub requested_downloads: Option<Vec<RequestedDownloads>>, - pub requested_entries: Option<Vec<u32>>, - pub requested_formats: Option<Vec<Format>>, - pub requested_subtitles: Option<HashMap<String, Subtitle>>, - pub resolution: Option<String>, - pub season: Option<String>, - pub season_number: Option<u32>, - pub series: Option<String>, - pub source_preference: Option<i32>, - pub sponsorblock_chapters: Option<Vec<SponsorblockChapter>>, - pub stretched_ratio: Option<Todo>, - pub subtitles: Option<HashMap<String, Vec<Caption>>>, - pub tags: Option<Vec<String>>, - pub tbr: Option<f64>, - pub thumbnail: Option<Url>, - pub thumbnails: Option<Vec<ThumbNail>>, - pub timestamp: Option<u64>, - pub title: Option<String>, - pub upload_date: Option<String>, - pub uploader: Option<String>, - pub uploader_id: Option<String>, - pub uploader_url: Option<String>, - pub url: Option<Url>, - pub vbr: Option<f64>, - pub vcodec: Option<String>, - pub video_ext: Option<String>, - pub view_count: Option<u32>, - pub was_live: Option<bool>, - pub webpage_url: Option<Url>, - pub webpage_url_basename: Option<String>, - pub webpage_url_domain: Option<String>, - pub width: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -#[allow(missing_copy_implementations)] -pub struct FilesToMove {} - -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct RequestedDownloads { - pub __files_to_merge: Option<Vec<Todo>>, - pub __finaldir: PathBuf, - pub __infojson_filename: PathBuf, - pub __postprocessors: Vec<Todo>, - pub __real_download: bool, - pub __write_download_archive: bool, - pub _filename: PathBuf, - pub _type: InfoType, - pub _version: Version, - pub abr: f64, - pub acodec: String, - pub aspect_ratio: Option<f64>, - pub asr: Option<u32>, - pub audio_channels: Option<u32>, - pub audio_ext: Option<String>, - pub chapters: Option<Vec<SponsorblockChapter>>, - pub duration: Option<f64>, - pub dynamic_range: Option<String>, - pub ext: String, - pub filename: PathBuf, - pub filepath: PathBuf, - pub filesize_approx: Option<u64>, - pub format: String, - pub format_id: String, - pub format_note: String, - pub fps: Option<f64>, - pub has_drm: Option<bool>, - pub height: Option<u32>, - pub http_headers: Option<HttpHeader>, - pub infojson_filename: PathBuf, - pub language: Option<String>, - pub manifest_url: Option<Url>, - pub protocol: String, - pub requested_formats: Option<Vec<Format>>, - pub resolution: String, - pub tbr: f64, - pub url: Option<Url>, - pub vbr: f64, - pub vcodec: String, - pub video_ext: Option<String>, - pub width: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Subtitle { - pub ext: SubtitleExt, - pub filepath: PathBuf, - pub filesize: Option<u64>, - pub fragment_base_url: Option<Url>, - pub fragments: Option<Vec<Fragment>>, - pub manifest_url: Option<Url>, - pub name: Option<String>, - pub protocol: Option<Todo>, - pub url: Url, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -pub enum SubtitleExt { - #[serde(alias = "vtt")] - Vtt, - - #[serde(alias = "mp4")] - Mp4, - - #[serde(alias = "json")] - Json, - #[serde(alias = "json3")] - Json3, - - #[serde(alias = "ttml")] - Ttml, - - #[serde(alias = "srv1")] - Srv1, - #[serde(alias = "srv2")] - Srv2, - #[serde(alias = "srv3")] - Srv3, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Caption { - pub ext: SubtitleExt, - pub filepath: Option<PathBuf>, - pub filesize: Option<u64>, - pub fragments: Option<Vec<SubtitleFragment>>, - pub fragment_base_url: Option<Url>, - pub manifest_url: Option<Url>, - pub name: Option<String>, - pub protocol: Option<String>, - pub url: String, - pub video_id: Option<String>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct SubtitleFragment { - path: PathBuf, - duration: Option<f64>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Chapter { - pub end_time: f64, - pub start_time: f64, - pub title: String, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq)] -#[serde(deny_unknown_fields)] -pub struct SponsorblockChapter { - /// This is an utterly useless field, and should thus be ignored - pub _categories: Option<Vec<Vec<Value>>>, - - pub categories: Option<Vec<SponsorblockChapterCategory>>, - pub category: Option<SponsorblockChapterCategory>, - pub category_names: Option<Vec<String>>, - pub end_time: f64, - pub name: Option<String>, - pub r#type: Option<SponsorblockChapterType>, - pub start_time: f64, - pub title: String, -} - -pub fn get_none<'de, D, T>(_: D) -> Result<Option<T>, D::Error> -where - D: Deserializer<'de>, -{ - Ok(None) -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub enum SponsorblockChapterType { - #[serde(alias = "skip")] - Skip, - - #[serde(alias = "chapter")] - Chapter, - - #[serde(alias = "poi")] - Poi, -} -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub enum SponsorblockChapterCategory { - #[serde(alias = "filler")] - Filler, - - #[serde(alias = "interaction")] - Interaction, - - #[serde(alias = "music_offtopic")] - MusicOfftopic, - - #[serde(alias = "poi_highlight")] - PoiHighlight, - - #[serde(alias = "preview")] - Preview, - - #[serde(alias = "sponsor")] - Sponsor, - - #[serde(alias = "selfpromo")] - SelfPromo, - - #[serde(alias = "chapter")] - Chapter, - - #[serde(alias = "intro")] - Intro, - - #[serde(alias = "outro")] - Outro, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -#[allow(missing_copy_implementations)] -pub struct HeatMapEntry { - pub start_time: f64, - pub end_time: f64, - pub value: f64, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd, Ord, Eq, Clone, Copy)] -#[serde(deny_unknown_fields)] -pub enum InfoType { - #[serde(alias = "playlist")] - Playlist, - - #[serde(alias = "url")] - Url, - - #[serde(alias = "video")] - Video, -} - -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct Version { - pub current_git_head: Option<String>, - pub release_git_head: String, - pub repository: String, - pub version: String, -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(from = "String")] -#[serde(deny_unknown_fields)] -pub enum Parent { - Root, - Id(String), -} - -impl Parent { - #[must_use] - pub fn id(&self) -> Option<&str> { - if let Self::Id(id) = self { - Some(id) - } else { - None - } - } -} - -impl From<String> for Parent { - fn from(value: String) -> Self { - if value == "root" { - Self::Root - } else { - Self::Id(value) - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(from = "String")] -#[serde(deny_unknown_fields)] -pub struct Id { - pub id: String, -} -impl From<String> for Id { - fn from(value: String) -> Self { - Self { - // Take the last element if the string is split with dots, otherwise take the full id - id: value.split('.').last().unwrap_or(&value).to_owned(), - } - } -} - -#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -#[allow(clippy::struct_excessive_bools)] -pub struct Comment { - pub id: Id, - pub text: String, - #[serde(default = "zero")] - pub like_count: u32, - pub is_pinned: bool, - pub author_id: String, - #[serde(default = "unknown")] - pub author: String, - pub author_is_verified: bool, - pub author_thumbnail: Url, - pub parent: Parent, - #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] - pub edited: bool, - // Can't also be deserialized, as it's already used in 'edited' - // _time_text: String, - pub timestamp: i64, - pub author_url: Url, - pub author_is_uploader: bool, - pub is_favorited: bool, -} -fn unknown() -> String { - "<Unknown>".to_string() -} -fn zero() -> u32 { - 0 -} -fn edited_from_time_text<'de, D>(d: D) -> Result<bool, D::Error> -where - D: Deserializer<'de>, -{ - let s = String::deserialize(d)?; - if s.contains(" (edited)") { - Ok(true) - } else { - Ok(false) - } -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct ThumbNail { - pub id: Option<String>, - pub preference: Option<i32>, - /// in the form of "[`height`]x[`width`]" - pub resolution: Option<String>, - pub url: Url, - pub width: Option<u32>, - pub height: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Format { - pub __needs_testing: Option<bool>, - pub __working: Option<bool>, - pub abr: Option<f64>, - pub acodec: Option<String>, - pub aspect_ratio: Option<f64>, - pub asr: Option<f64>, - pub audio_channels: Option<u32>, - pub audio_ext: Option<String>, - pub columns: Option<u32>, - pub container: Option<String>, - pub downloader_options: Option<DownloaderOptions>, - pub dynamic_range: Option<String>, - pub ext: String, - pub filepath: Option<PathBuf>, - pub filesize: Option<u64>, - pub filesize_approx: Option<u64>, - pub format: Option<String>, - pub format_id: String, - pub format_index: Option<String>, - pub format_note: Option<String>, - pub fps: Option<f64>, - pub fragment_base_url: Option<Todo>, - pub fragments: Option<Vec<Fragment>>, - pub has_drm: Option<bool>, - pub height: Option<u32>, - pub http_headers: Option<HttpHeader>, - pub is_dash_periods: Option<bool>, - pub language: Option<String>, - pub language_preference: Option<i32>, - pub manifest_stream_number: Option<u32>, - pub manifest_url: Option<Url>, - pub preference: Option<i32>, - pub protocol: Option<String>, - pub quality: Option<f64>, - pub resolution: Option<String>, - pub rows: Option<u32>, - pub source_preference: Option<i32>, - pub tbr: Option<f64>, - pub url: Url, - pub vbr: Option<f64>, - pub vcodec: String, - pub video_ext: Option<String>, - pub width: Option<u32>, -} - -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -#[allow(missing_copy_implementations)] -pub struct DownloaderOptions { - http_chunk_size: u64, -} - -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, PartialOrd, Ord)] -#[serde(deny_unknown_fields)] -pub struct HttpHeader { - #[serde(alias = "User-Agent")] - pub user_agent: Option<String>, - - #[serde(alias = "Accept")] - pub accept: Option<String>, - - #[serde(alias = "X-Forwarded-For")] - pub x_forwarded_for: Option<String>, - - #[serde(alias = "Accept-Language")] - pub accept_language: Option<String>, - - #[serde(alias = "Sec-Fetch-Mode")] - pub sec_fetch_mode: Option<String>, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(deny_unknown_fields)] -pub struct Fragment { - pub url: Option<Url>, - pub duration: Option<f64>, - pub path: Option<PathBuf>, -} - -impl InfoJson { - pub fn to_py_dict(self, py: Python<'_>) -> PyResult<Bound<'_, PyDict>> { - let output: Bound<'_, PyDict> = json_loads_str(py, self)?; - Ok(output) - } -} diff --git a/crates/yt_dlp/src/wrapper/yt_dlp_options.rs b/crates/yt_dlp/src/wrapper/yt_dlp_options.rs deleted file mode 100644 index c2a86df..0000000 --- a/crates/yt_dlp/src/wrapper/yt_dlp_options.rs +++ /dev/null @@ -1,62 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -use pyo3::{types::PyDict, Bound, PyResult, Python}; -use serde::Serialize; - -use crate::json_loads; - -#[derive(Serialize, Clone)] -pub struct YtDlpOptions { - pub playliststart: u32, - pub playlistend: u32, - pub noplaylist: bool, - pub extract_flat: ExtractFlat, - // pub extractor_args: ExtractorArgs, - // pub format: String, - // pub fragment_retries: u32, - // #[serde(rename(serialize = "getcomments"))] - // pub get_comments: bool, - // #[serde(rename(serialize = "ignoreerrors"))] - // pub ignore_errors: bool, - // pub retries: u32, - // #[serde(rename(serialize = "writeinfojson"))] - // pub write_info_json: bool, - // pub postprocessors: Vec<serde_json::Map<String, serde_json::Value>>, -} - -#[derive(Serialize, Copy, Clone)] -pub enum ExtractFlat { - #[serde(rename(serialize = "in_playlist"))] - InPlaylist, - - #[serde(rename(serialize = "discard_in_playlist"))] - DiscardInPlaylist, -} - -#[derive(Serialize, Clone)] -pub struct ExtractorArgs { - pub youtube: YoutubeExtractorArgs, -} - -#[derive(Serialize, Clone)] -pub struct YoutubeExtractorArgs { - comment_sort: Vec<String>, - max_comments: Vec<String>, -} - -impl YtDlpOptions { - pub fn to_py_dict(self, py: Python) -> PyResult<Bound<PyDict>> { - let string = serde_json::to_string(&self).expect("This should always work"); - - let output: Bound<PyDict> = json_loads(py, string)?; - Ok(output) - } -} diff --git a/crates/yt_dlp/update.sh b/crates/yt_dlp/update.sh index c1a0215..ab03b62 100755 --- a/crates/yt_dlp/update.sh +++ b/crates/yt_dlp/update.sh @@ -10,6 +10,4 @@ # You should have received a copy of the License along with this program. # If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -cd "$(dirname "$0")" || exit 1 -[ "$1" = "upgrade" ] && cargo upgrade --incompatible -cargo update +./crates/pyo3-pylogger/update.sh "$@" |