diff options
Diffstat (limited to 'crates')
55 files changed, 7415 insertions, 0 deletions
diff --git a/crates/yt/Cargo.toml b/crates/yt/Cargo.toml new file mode 100644 index 0000000..17d4016 --- /dev/null +++ b/crates/yt/Cargo.toml @@ -0,0 +1,65 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +[package] +name = "yt" +description = "A fully featured command line YouTube client" +keywords = [] +categories = [] +default-run = "yt" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +anyhow = "1.0.98" +blake3 = "1.8.2" +chrono = { version = "0.4.41", features = ["now"] } +chrono-humanize = "0.2.3" +clap = { version = "4.5.40", features = ["derive"] } +futures = "0.3.31" +nucleo-matcher = "0.3.1" +owo-colors = "4.2.1" +regex = "1.11.1" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "sqlite"] } +stderrlog = "0.6.0" +tempfile = "3.20.0" +toml = "0.8.23" +trinitry = { version = "0.2.2" } +xdg = "3.0.0" +bytes.workspace = true +libmpv2.workspace = true +log.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +url.workspace = true +yt_dlp.workspace = true +termsize.workspace = true +uu_fmt.workspace = true +notify = { version = "8.0.0", default-features = false } + +[[bin]] +name = "yt" +doc = false +path = "src/main.rs" + +[dev-dependencies] + +[lints] +workspace = true + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/yt/src/ansi_escape_codes.rs b/crates/yt/src/ansi_escape_codes.rs new file mode 100644 index 0000000..ae1805d --- /dev/null +++ b/crates/yt/src/ansi_escape_codes.rs @@ -0,0 +1,26 @@ +// see: https://en.wikipedia.org/wiki/ANSI_escape_code#Control_Sequence_Introducer_commands +const CSI: &str = "\x1b["; +pub fn erase_in_display_from_cursor() { + print!("{CSI}0J"); +} +pub fn cursor_up(number: usize) { + // HACK(@bpeetz): The default is `1` and running this command with a + // number of `0` results in it using the default (i.e., `1`) <2025-03-25> + if number != 0 { + print!("{CSI}{number}A"); + } +} + +pub fn clear_whole_line() { + eprint!("{CSI}2K"); +} +pub fn move_to_col(x: usize) { + eprint!("{CSI}{x}G"); +} + +pub fn hide_cursor() { + eprint!("{CSI}?25l"); +} +pub fn show_cursor() { + eprint!("{CSI}?25h"); +} diff --git a/crates/yt/src/app.rs b/crates/yt/src/app.rs new file mode 100644 index 0000000..15a9388 --- /dev/null +++ b/crates/yt/src/app.rs @@ -0,0 +1,50 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::{Context, Result}; +use log::warn; +use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; + +use crate::{config::Config, storage::migrate::migrate_db}; + +#[derive(Debug)] +pub struct App { + pub database: SqlitePool, + pub config: Config, +} + +impl App { + pub async fn new(config: Config, should_migrate_db: bool) -> Result<Self> { + let options = SqliteConnectOptions::new() + .filename(&config.paths.database_path) + .optimize_on_close(true, None) + .create_if_missing(true); + + let pool = SqlitePool::connect_with(options) + .await + .context("Failed to connect to database!")?; + + let app = App { + database: pool, + config, + }; + + if should_migrate_db { + migrate_db(&app) + .await + .context("Failed to migrate db to new version")?; + } else { + warn!("Skipping database migration."); + } + + Ok(app) + } +} diff --git a/crates/yt/src/cache/mod.rs b/crates/yt/src/cache/mod.rs new file mode 100644 index 0000000..83d5ee0 --- /dev/null +++ b/crates/yt/src/cache/mod.rs @@ -0,0 +1,105 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::{Context, Result}; +use log::{debug, info}; +use tokio::fs; + +use crate::{ + app::App, + storage::video_database::{ + Video, VideoStatus, VideoStatusMarker, downloader::set_video_cache_path, get, + }, +}; + +async fn invalidate_video(app: &App, video: &Video, hard: bool) -> Result<()> { + info!("Invalidating cache of video: '{}'", video.title); + + if hard { + if let VideoStatus::Cached { + cache_path: path, .. + } = &video.status + { + info!("Removing cached video at: '{}'", path.display()); + if let Err(err) = fs::remove_file(path).await.map_err(|err| err.kind()) { + match err { + std::io::ErrorKind::NotFound => { + // The path is already gone + debug!( + "Not actually removing path: '{}'. It is already gone.", + path.display() + ); + } + err => Err(std::io::Error::from(err)).with_context(|| { + format!( + "Failed to delete video ('{}') cache path: '{}'.", + video.title, + path.display() + ) + })?, + } + } + } + } + + set_video_cache_path(app, &video.extractor_hash, None).await?; + + Ok(()) +} + +pub async fn invalidate(app: &App, hard: bool) -> Result<()> { + let all_cached_things = get::videos(app, &[VideoStatusMarker::Cached]).await?; + + info!("Got videos to invalidate: '{}'", all_cached_things.len()); + + for video in all_cached_things { + invalidate_video(app, &video, hard).await?; + } + + Ok(()) +} + +/// # Panics +/// Only if internal assertions fail. +pub async fn maintain(app: &App, all: bool) -> Result<()> { + let domain = if all { + VideoStatusMarker::ALL.as_slice() + } else { + &[VideoStatusMarker::Watch, VideoStatusMarker::Cached] + }; + + let cached_videos = get::videos(app, domain).await?; + + let mut found_focused = 0; + for vid in cached_videos { + if let VideoStatus::Cached { + cache_path: path, + is_focused, + } = &vid.status + { + info!("Checking if path ('{}') exists", path.display()); + if !path.exists() { + invalidate_video(app, &vid, false).await?; + } + + if *is_focused { + found_focused += 1; + } + } + } + + assert!( + found_focused <= 1, + "Only one video can be focused at a time" + ); + + Ok(()) +} diff --git a/crates/yt/src/cli.rs b/crates/yt/src/cli.rs new file mode 100644 index 0000000..de7a5b8 --- /dev/null +++ b/crates/yt/src/cli.rs @@ -0,0 +1,371 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{path::PathBuf, str::FromStr}; + +use anyhow::Context; +use bytes::Bytes; +use chrono::NaiveDate; +use clap::{ArgAction, Args, Parser, Subcommand}; +use url::Url; + +use crate::{ + select::selection_file::duration::MaybeDuration, + storage::video_database::extractor_hash::LazyExtractorHash, +}; + +#[derive(Parser, Debug)] +#[clap(author, about, long_about = None)] +#[allow(clippy::module_name_repetitions)] +/// An command line interface to select, download and watch videos +pub struct CliArgs { + #[command(subcommand)] + /// The subcommand to execute [default: select] + pub command: Option<Command>, + + /// Show the version and exit + #[arg(long, short = 'V', action= ArgAction::SetTrue)] + pub version: bool, + + /// Do not perform database migration before starting. + /// Setting this could cause runtime database access errors. + #[arg(long, short, action=ArgAction::SetTrue, default_value_t = false)] + pub no_migrate_db: bool, + + /// Display colors [defaults to true, if the config file has no value] + #[arg(long, short = 'C')] + pub color: Option<bool>, + + /// Set the path to the videos.db. This overrides the default and the config file. + #[arg(long, short)] + pub db_path: Option<PathBuf>, + + /// Set the path to the config.toml. + /// This overrides the default. + #[arg(long, short)] + pub config_path: Option<PathBuf>, + + /// Increase message verbosity + #[arg(long="verbose", short = 'v', action = ArgAction::Count)] + pub verbosity: u8, + + /// Silence all output + #[arg(long, short = 'q')] + pub quiet: bool, +} + +#[derive(Subcommand, Debug)] +pub enum Command { + /// Download and cache URLs + Download { + /// Forcefully re-download all cached videos (i.e. delete the cache path, then download). + #[arg(short, long)] + force: bool, + + /// The maximum size the download dir should have. Beware that the value must be given in + /// bytes. + #[arg(short, long, value_parser = byte_parser)] + max_cache_size: Option<u64>, + }, + + /// Select, download and watch in one command. + Sedowa {}, + /// Download and watch in one command. + Dowa {}, + + /// Work with single videos + Videos { + #[command(subcommand)] + cmd: VideosCommand, + }, + + /// Watch the already cached (and selected) videos + Watch {}, + + /// Visualize the current playlist + Playlist { + /// Linger and display changes + #[arg(short, long)] + watch: bool, + }, + + /// Show, which videos have been selected to be watched (and their cache status) + Status {}, + + /// Show, the configuration options in effect + Config {}, + + /// Display the comments of the currently playing video + Comments {}, + /// Display the description of the currently playing video + Description {}, + + /// Manipulate the video cache in the database + #[command(visible_alias = "db")] + Database { + #[command(subcommand)] + command: CacheCommand, + }, + + /// Change the state of videos in the database (the default) + Select { + #[command(subcommand)] + cmd: Option<SelectCommand>, + }, + + /// Update the video database + Update { + #[arg(short, long)] + /// The number of videos to updating + max_backlog: Option<usize>, + + #[arg(short, long)] + /// The subscriptions to update (can be given multiple times) + subscriptions: Vec<String>, + }, + + /// Manipulate subscription + #[command(visible_alias = "subs")] + Subscriptions { + #[command(subcommand)] + cmd: SubscriptionCommand, + }, +} + +fn byte_parser(input: &str) -> Result<u64, anyhow::Error> { + Ok(input + .parse::<Bytes>() + .with_context(|| format!("Failed to parse '{input}' as bytes!"))? + .as_u64()) +} + +impl Default for Command { + fn default() -> Self { + Self::Select { + cmd: Some(SelectCommand::default()), + } + } +} + +#[derive(Subcommand, Clone, Debug)] +pub enum VideosCommand { + /// List the videos in the database + #[command(visible_alias = "ls")] + List { + /// An optional search query to limit the results + #[arg(action = ArgAction::Append)] + search_query: Option<String>, + + /// The number of videos to show + #[arg(short, long)] + limit: Option<usize>, + }, + + /// Get detailed information about a video + Info { + /// The short hash of the video + hash: LazyExtractorHash, + }, +} + +#[derive(Subcommand, Clone, Debug)] +pub enum SubscriptionCommand { + /// Subscribe to an URL + Add { + #[arg(short, long)] + /// The human readable name of the subscription + name: Option<String>, + + /// The URL to listen to + url: Url, + }, + + /// Unsubscribe from an URL + Remove { + /// The human readable name of the subscription + name: String, + }, + + /// Import a bunch of URLs as subscriptions. + Import { + /// The file containing the URLs. Will use Stdin otherwise. + file: Option<PathBuf>, + + /// Remove any previous subscriptions + #[arg(short, long)] + force: bool, + }, + /// Write all subscriptions in an format understood by `import` + Export {}, + + /// List all subscriptions + List {}, +} + +#[derive(Clone, Debug, Args)] +#[command(infer_subcommands = true)] +/// Mark the video given by the hash to be watched +pub struct SharedSelectionCommandArgs { + /// The ordering priority (higher means more at the top) + #[arg(short, long)] + pub priority: Option<i64>, + + /// The subtitles to download (e.g. 'en,de,sv') + #[arg(short = 'l', long)] + pub subtitle_langs: Option<String>, + + /// The speed to set mpv to + #[arg(short, long)] + pub speed: Option<f64>, + + /// The short extractor hash + pub hash: LazyExtractorHash, + + pub title: Option<String>, + + pub date: Option<OptionalNaiveDate>, + + pub publisher: Option<OptionalPublisher>, + + pub duration: Option<MaybeDuration>, + + pub url: Option<Url>, +} +#[derive(Clone, Debug, Copy)] +pub struct OptionalNaiveDate { + pub date: Option<NaiveDate>, +} +impl FromStr for OptionalNaiveDate { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No release date]" { + Ok(Self { date: None }) + } else { + Ok(Self { + date: Some(NaiveDate::from_str(v)?), + }) + } + } +} +#[derive(Clone, Debug)] +pub struct OptionalPublisher { + pub publisher: Option<String>, +} +impl FromStr for OptionalPublisher { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result<Self, Self::Err> { + if v == "[No author]" { + Ok(Self { publisher: None }) + } else { + Ok(Self { + publisher: Some(v.to_owned()), + }) + } + } +} + +#[derive(Subcommand, Clone, Debug)] +// NOTE: Keep this in sync with the [`constants::HELP_STR`] constant. <2024-08-20> +// NOTE: Also keep this in sync with the `tree-sitter-yts/grammar.js`. <2024-11-04> +pub enum SelectCommand { + /// Open a `git rebase` like file to select the videos to watch (the default) + File { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Use the last selection file (useful if you've spend time on it and want to get it again) + #[arg(long, short, conflicts_with = "done")] + use_last_selection: bool, + }, + + /// Add a video to the database + /// + /// This optionally supports to add a playlist. + /// When a playlist is added, the `start` and `stop` arguments can be used to select which + /// playlist entries to include. + #[command(visible_alias = "a")] + Add { + urls: Vec<Url>, + + /// Start adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 's', long)] + start: Option<usize>, + + /// Stop adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 'e', long)] + stop: Option<usize>, + }, + + /// Mark the video given by the hash to be watched + #[command(visible_alias = "w")] + Watch { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash to be dropped + #[command(visible_alias = "d")] + Drop { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash as already watched + #[command(visible_alias = "wd")] + Watched { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Open the video URL in Firefox's `timesinks.youtube` profile + #[command(visible_alias = "u")] + Url { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Reset the videos status to 'Pick' + #[command(visible_alias = "p")] + Pick { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, +} +impl Default for SelectCommand { + fn default() -> Self { + Self::File { + done: false, + use_last_selection: false, + } + } +} + +#[derive(Subcommand, Clone, Copy, Debug)] +pub enum CacheCommand { + /// Invalidate all cache entries + Invalidate { + /// Also delete the cache path + #[arg(short, long)] + hard: bool, + }, + + /// Perform basic maintenance operations on the database. + /// This helps recovering from invalid db states after a crash (or force exit via <CTRL-C>). + /// + /// 1. Check every path for validity (removing all invalid cache entries) + #[command(verbatim_doc_comment)] + Maintain { + /// Check every video (otherwise only the videos to be watched are checked) + #[arg(short, long)] + all: bool, + }, +} diff --git a/crates/yt/src/comments/comment.rs b/crates/yt/src/comments/comment.rs new file mode 100644 index 0000000..5bc939c --- /dev/null +++ b/crates/yt/src/comments/comment.rs @@ -0,0 +1,152 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use serde::{Deserialize, Deserializer, Serialize}; +use url::Url; + +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub enum Parent { + Root, + Id(String), +} + +impl Parent { + #[must_use] + pub fn id(&self) -> Option<&str> { + if let Self::Id(id) = self { + Some(id) + } else { + None + } + } +} + +impl From<String> for Parent { + fn from(value: String) -> Self { + if value == "root" { + Self::Root + } else { + Self::Id(value) + } + } +} + +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub struct Id { + pub id: String, +} +impl From<String> for Id { + fn from(value: String) -> Self { + Self { + // Take the last element if the string is split with dots, otherwise take the full id + id: value.split('.').last().unwrap_or(&value).to_owned(), + } + } +} + +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[allow(clippy::struct_excessive_bools)] +pub struct Comment { + pub id: Id, + pub text: String, + #[serde(default = "zero")] + pub like_count: u32, + pub is_pinned: bool, + pub author_id: String, + #[serde(default = "unknown")] + pub author: String, + pub author_is_verified: bool, + pub author_thumbnail: Url, + pub parent: Parent, + #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] + pub edited: bool, + // Can't also be deserialized, as it's already used in 'edited' + // _time_text: String, + pub timestamp: i64, + pub author_url: Option<Url>, + pub author_is_uploader: bool, + pub is_favorited: bool, +} + +fn unknown() -> String { + "<Unknown>".to_string() +} +fn zero() -> u32 { + 0 +} +fn edited_from_time_text<'de, D>(d: D) -> Result<bool, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(d)?; + if s.contains(" (edited)") { + Ok(true) + } else { + Ok(false) + } +} + +#[derive(Debug, Clone)] +#[allow(clippy::module_name_repetitions)] +pub struct CommentExt { + pub value: Comment, + pub replies: Vec<CommentExt>, +} + +#[derive(Debug, Default)] +pub struct Comments { + pub(super) vec: Vec<CommentExt>, +} + +impl Comments { + pub fn new() -> Self { + Self::default() + } + pub fn push(&mut self, value: CommentExt) { + self.vec.push(value); + } + pub fn get_mut(&mut self, key: &str) -> Option<&mut CommentExt> { + self.vec.iter_mut().filter(|c| c.value.id.id == key).last() + } + pub fn insert(&mut self, key: &str, value: CommentExt) { + let parent = self + .vec + .iter_mut() + .filter(|c| c.value.id.id == key) + .last() + .expect("One of these should exist"); + parent.push_reply(value); + } +} +impl CommentExt { + pub fn push_reply(&mut self, value: CommentExt) { + self.replies.push(value); + } + pub fn get_mut_reply(&mut self, key: &str) -> Option<&mut CommentExt> { + self.replies + .iter_mut() + .filter(|c| c.value.id.id == key) + .last() + } +} + +impl From<Comment> for CommentExt { + fn from(value: Comment) -> Self { + Self { + replies: vec![], + value, + } + } +} diff --git a/crates/yt/src/comments/description.rs b/crates/yt/src/comments/description.rs new file mode 100644 index 0000000..e8cb29d --- /dev/null +++ b/crates/yt/src/comments/description.rs @@ -0,0 +1,46 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + App, + comments::output::display_fmt_and_less, + storage::video_database::{Video, get}, + unreachable::Unreachable, +}; + +use anyhow::{Result, bail}; +use yt_dlp::{InfoJson, json_cast}; + +pub async fn description(app: &App) -> Result<()> { + let description = get(app).await?; + display_fmt_and_less(description).await?; + + Ok(()) +} + +pub async fn get(app: &App) -> Result<String> { + let currently_playing_video: Video = + if let Some(video) = get::currently_focused_video(app).await? { + video + } else { + bail!("Could not find a currently playing video!"); + }; + + let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( + "A currently *playing* must be cached. And thus the info.json should be available", + ); + + Ok(info_json + .get("description") + .map(|val| json_cast!(val, as_str)) + .unwrap_or("<No description>") + .to_owned()) +} diff --git a/crates/yt/src/comments/display.rs b/crates/yt/src/comments/display.rs new file mode 100644 index 0000000..6166b2b --- /dev/null +++ b/crates/yt/src/comments/display.rs @@ -0,0 +1,118 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::fmt::Write; + +use chrono::{Local, TimeZone}; +use chrono_humanize::{Accuracy, HumanTime, Tense}; + +use crate::comments::comment::CommentExt; + +use super::comment::Comments; + +impl Comments { + pub fn render(&self, color: bool) -> String { + self.render_help(color).expect("This should never fail.") + } + + fn render_help(&self, color: bool) -> Result<String, std::fmt::Error> { + macro_rules! c { + ($color_str:expr, $write:ident, $color:expr) => { + if $color { + $write.write_str(concat!("\x1b[", $color_str, "m"))? + } + }; + } + + fn format( + comment: &CommentExt, + f: &mut String, + ident_count: u32, + color: bool, + ) -> std::fmt::Result { + let ident = &(0..ident_count).map(|_| " ").collect::<String>(); + let value = &comment.value; + + f.write_str(ident)?; + + if value.author_is_uploader { + c!("91;1", f, color); + } else { + c!("35", f, color); + } + + f.write_str(&value.author)?; + c!("0", f, color); + if value.edited || value.is_favorited { + f.write_str("[")?; + if value.edited { + f.write_str("")?; + } + if value.edited && value.is_favorited { + f.write_str(" ")?; + } + if value.is_favorited { + f.write_str("")?; + } + f.write_str("]")?; + } + + c!("36;1", f, color); + write!( + f, + " {}", + HumanTime::from( + Local + .timestamp_opt(value.timestamp, 0) + .single() + .expect("This should be valid") + ) + .to_text_en(Accuracy::Rough, Tense::Past) + )?; + c!("0", f, color); + + // c!("31;1", f); + // f.write_fmt(format_args!(" [{}]", comment.value.like_count))?; + // c!("0", f); + + f.write_str(":\n")?; + f.write_str(ident)?; + + f.write_str(&value.text.replace('\n', &format!("\n{ident}")))?; + f.write_str("\n")?; + + if comment.replies.is_empty() { + f.write_str("\n")?; + } else { + let mut children = comment.replies.clone(); + children.sort_by(|a, b| a.value.timestamp.cmp(&b.value.timestamp)); + + for child in children { + format(&child, f, ident_count + 4, color)?; + } + } + + Ok(()) + } + + let mut f = String::new(); + + if !&self.vec.is_empty() { + let mut children = self.vec.clone(); + children.sort_by(|a, b| b.value.like_count.cmp(&a.value.like_count)); + + for child in children { + format(&child, &mut f, 0, color)?; + } + } + Ok(f) + } +} diff --git a/crates/yt/src/comments/mod.rs b/crates/yt/src/comments/mod.rs new file mode 100644 index 0000000..876146d --- /dev/null +++ b/crates/yt/src/comments/mod.rs @@ -0,0 +1,167 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::mem; + +use anyhow::{Result, bail}; +use comment::{Comment, CommentExt, Comments, Parent}; +use output::display_fmt_and_less; +use regex::Regex; +use yt_dlp::{InfoJson, json_cast}; + +use crate::{ + app::App, + storage::video_database::{Video, get}, + unreachable::Unreachable, +}; + +mod comment; +mod display; +pub mod output; + +pub mod description; +pub use description::*; + +#[allow(clippy::too_many_lines)] +pub async fn get(app: &App) -> Result<Comments> { + let currently_playing_video: Video = + if let Some(video) = get::currently_focused_video(app).await? { + video + } else { + bail!("Could not find a currently playing video!"); + }; + + let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( + "A currently *playing* video must be cached. And thus the info.json should be available", + ); + + let base_comments = if let Some(comments) = info_json.get("comments") { + json_cast!(comments, as_array) + } else { + bail!( + "The video ('{}') does not have comments!", + info_json + .get("title") + .map(|val| json_cast!(val, as_str)) + .unwrap_or("<No Title>") + ) + }; + + let mut comments = Comments::new(); + for c in base_comments { + let c: Comment = serde_json::from_value(c.to_owned())?; + if let Parent::Id(id) = &c.parent { + comments.insert(&(id.clone()), CommentExt::from(c)); + } else { + comments.push(CommentExt::from(c)); + } + } + + comments.vec.iter_mut().for_each(|comment| { + let replies = mem::take(&mut comment.replies); + let mut output_replies: Vec<CommentExt> = vec![]; + + let re = Regex::new(r"\u{200b}?(@[^\t\s]+)\u{200b}?").unreachable("This is hardcoded"); + for reply in replies { + if let Some(replyee_match) = re.captures(&reply.value.text){ + let full_match = replyee_match.get(0).unreachable("This will always exist"); + let text = reply. + value. + text[0..full_match.start()] + .to_owned() + + + &reply + .value + .text[full_match.end()..]; + let text: &str = text.trim().trim_matches('\u{200b}'); + + let replyee = replyee_match.get(1).unreachable("This should also exist").as_str(); + + + if let Some(parent) = output_replies + .iter_mut() + // .rev() + .flat_map(|com| &mut com.replies) + .flat_map(|com| &mut com.replies) + .flat_map(|com| &mut com.replies) + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else if let Some(parent) = output_replies + .iter_mut() + // .rev() + .flat_map(|com| &mut com.replies) + .flat_map(|com| &mut com.replies) + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else if let Some(parent) = output_replies + .iter_mut() + // .rev() + .flat_map(|com| &mut com.replies) + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else if let Some(parent) = output_replies.iter_mut() + // .rev() + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else { + eprintln!( + "Failed to find a parent for ('{}') both directly and via replies! The reply text was:\n'{}'\n", + replyee, + reply.value.text + ); + output_replies.push(reply); + } + } else { + output_replies.push(reply); + } + } + comment.replies = output_replies; + }); + + Ok(comments) +} + +pub async fn comments(app: &App) -> Result<()> { + let comments = get(app).await?; + + display_fmt_and_less(comments.render(true)).await?; + + Ok(()) +} + +#[cfg(test)] +mod test { + #[test] + fn test_string_replacement() { + let s = "A \n\nB\n\nC".to_owned(); + assert_eq!("A \n \n B\n \n C", s.replace('\n', "\n ")); + } +} diff --git a/crates/yt/src/comments/output.rs b/crates/yt/src/comments/output.rs new file mode 100644 index 0000000..cb3a9c4 --- /dev/null +++ b/crates/yt/src/comments/output.rs @@ -0,0 +1,53 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + io::Write, + process::{Command, Stdio}, +}; + +use anyhow::{Context, Result}; +use uu_fmt::{FmtOptions, process_text}; + +use crate::unreachable::Unreachable; + +pub async fn display_fmt_and_less(input: String) -> Result<()> { + let mut less = Command::new("less") + .args(["--raw-control-chars"]) + .stdin(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .context("Failed to run less")?; + + let input = format_text(&input); + let mut stdin = less.stdin.take().context("Failed to open stdin")?; + std::thread::spawn(move || { + stdin + .write_all(input.as_bytes()) + .unreachable("Should be able to write to the stdin of less"); + }); + + let _ = less.wait().context("Failed to await less")?; + + Ok(()) +} + +#[must_use] +pub fn format_text(input: &str) -> String { + let width = termsize::get().map_or(90, |size| size.cols); + let fmt_opts = FmtOptions { + uniform: true, + split_only: true, + ..FmtOptions::new(Some(width as usize), None, Some(4)) + }; + + process_text(input, &fmt_opts) +} diff --git a/crates/yt/src/config/default.rs b/crates/yt/src/config/default.rs new file mode 100644 index 0000000..4ed643b --- /dev/null +++ b/crates/yt/src/config/default.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::PathBuf; + +use anyhow::{Context, Result}; + +fn get_runtime_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_runtime_file(name) + .with_context(|| format!("Failed to place runtime file: '{name}'")) +} +fn get_data_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_data_file(name) + .with_context(|| format!("Failed to place data file: '{name}'")) +} +fn get_config_path(name: &'static str) -> Result<PathBuf> { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_config_file(name) + .with_context(|| format!("Failed to place config file: '{name}'")) +} + +pub(super) fn create_path(path: PathBuf) -> Result<PathBuf> { + if !path.exists() { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + } + + Ok(path) +} + +pub(crate) const PREFIX: &str = "yt"; + +pub(crate) mod global { + pub(crate) fn display_colors() -> bool { + // TODO: This should probably check if the output is a tty and otherwise return `false` <2025-02-14> + true + } +} + +pub(crate) mod select { + pub(crate) fn playback_speed() -> f64 { + 2.7 + } + pub(crate) fn subtitle_langs() -> &'static str { + "" + } +} + +pub(crate) mod watch { + pub(crate) fn local_displays_length() -> usize { + 1000 + } +} + +pub(crate) mod update { + pub(crate) fn max_backlog() -> usize { + 20 + } +} + +pub(crate) mod paths { + use std::{env::temp_dir, path::PathBuf}; + + use anyhow::Result; + + use super::{PREFIX, create_path, get_config_path, get_data_path, get_runtime_path}; + + // We download to the temp dir to avoid taxing the disk + pub(crate) fn download_dir() -> Result<PathBuf> { + let temp_dir = temp_dir(); + + create_path(temp_dir.join(PREFIX)) + } + pub(crate) fn mpv_config_path() -> Result<PathBuf> { + get_config_path("mpv.conf") + } + pub(crate) fn mpv_input_path() -> Result<PathBuf> { + get_config_path("mpv.input.conf") + } + pub(crate) fn database_path() -> Result<PathBuf> { + get_data_path("videos.sqlite") + } + pub(crate) fn config_path() -> Result<PathBuf> { + get_config_path("config.toml") + } + pub(crate) fn last_selection_path() -> Result<PathBuf> { + get_runtime_path("selected.yts") + } +} + +pub(crate) mod download { + pub(crate) fn max_cache_size() -> &'static str { + "3 GiB" + } +} diff --git a/crates/yt/src/config/definitions.rs b/crates/yt/src/config/definitions.rs new file mode 100644 index 0000000..ce8c0d4 --- /dev/null +++ b/crates/yt/src/config/definitions.rs @@ -0,0 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::PathBuf; + +use serde::Deserialize; + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ConfigFile { + pub global: Option<GlobalConfig>, + pub select: Option<SelectConfig>, + pub watch: Option<WatchConfig>, + pub paths: Option<PathsConfig>, + pub download: Option<DownloadConfig>, + pub update: Option<UpdateConfig>, +} + +#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] +#[serde(deny_unknown_fields)] +pub(crate) struct GlobalConfig { + pub display_colors: Option<bool>, +} + +#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] +#[serde(deny_unknown_fields)] +pub(crate) struct UpdateConfig { + pub max_backlog: Option<usize>, +} + +#[derive(Debug, Deserialize, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub(crate) struct DownloadConfig { + /// This will then be converted to an u64 + pub max_cache_size: Option<String>, +} + +#[derive(Debug, Deserialize, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub(crate) struct SelectConfig { + pub playback_speed: Option<f64>, + pub subtitle_langs: Option<String>, +} + +#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] +#[serde(deny_unknown_fields)] +pub(crate) struct WatchConfig { + pub local_displays_length: Option<usize>, +} + +#[derive(Debug, Deserialize, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub(crate) struct PathsConfig { + pub download_dir: Option<PathBuf>, + pub mpv_config_path: Option<PathBuf>, + pub mpv_input_path: Option<PathBuf>, + pub database_path: Option<PathBuf>, + pub last_selection_path: Option<PathBuf>, +} diff --git a/crates/yt/src/config/file_system.rs b/crates/yt/src/config/file_system.rs new file mode 100644 index 0000000..2463e9d --- /dev/null +++ b/crates/yt/src/config/file_system.rs @@ -0,0 +1,120 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::config::{DownloadConfig, PathsConfig, SelectConfig, WatchConfig}; + +use super::{ + Config, GlobalConfig, UpdateConfig, + default::{create_path, download, global, paths, select, update, watch}, +}; + +use std::{fs::read_to_string, path::PathBuf}; + +use anyhow::{Context, Result}; +use bytes::Bytes; + +macro_rules! get { + ($default:path, $config:expr, $key_one:ident, $($keys:ident),*) => { + { + let maybe_value = get!{@option $config, $key_one, $($keys),*}; + if let Some(value) = maybe_value { + value + } else { + $default().to_owned() + } + } + }; + + (@option $config:expr, $key_one:ident, $($keys:ident),*) => { + if let Some(key) = $config.$key_one.clone() { + get!{@option key, $($keys),*} + } else { + None + } + }; + (@option $config:expr, $key_one:ident) => { + $config.$key_one + }; + + (@path_if_none $config:expr, $option_default:expr, $default:path, $key_one:ident, $($keys:ident),*) => { + { + let maybe_download_dir: Option<PathBuf> = + get! {@option $config, $key_one, $($keys),*}; + + let down_dir = if let Some(dir) = maybe_download_dir { + PathBuf::from(dir) + } else { + if let Some(path) = $option_default { + path + } else { + $default() + .with_context(|| format!("Failed to get default path for: '{}.{}'", stringify!($key_one), stringify!($($keys),*)))? + } + }; + create_path(down_dir)? + } + }; + (@path $config:expr, $default:path, $key_one:ident, $($keys:ident),*) => { + get! {@path_if_none $config, None, $default, $key_one, $($keys),*} + }; +} + +impl Config { + pub fn from_config_file( + db_path: Option<PathBuf>, + config_path: Option<PathBuf>, + display_colors: Option<bool>, + ) -> Result<Self> { + let config_file_path = + config_path.map_or_else(|| -> Result<_> { paths::config_path() }, Ok)?; + + let config: super::definitions::ConfigFile = + toml::from_str(&read_to_string(config_file_path).unwrap_or(String::new())) + .context("Failed to parse the config file as toml")?; + + Ok(Self { + global: GlobalConfig { + display_colors: { + let config_value: Option<bool> = get! {@option config, global, display_colors}; + + display_colors.unwrap_or(config_value.unwrap_or_else(global::display_colors)) + }, + }, + select: SelectConfig { + playback_speed: get! {select::playback_speed, config, select, playback_speed}, + subtitle_langs: get! {select::subtitle_langs, config, select, subtitle_langs}, + }, + watch: WatchConfig { + local_displays_length: get! {watch::local_displays_length, config, watch, local_displays_length}, + }, + update: UpdateConfig { + max_backlog: get! {update::max_backlog, config, update, max_backlog}, + }, + paths: PathsConfig { + download_dir: get! {@path config, paths::download_dir, paths, download_dir}, + mpv_config_path: get! {@path config, paths::mpv_config_path, paths, mpv_config_path}, + mpv_input_path: get! {@path config, paths::mpv_input_path, paths, mpv_input_path}, + database_path: get! {@path_if_none config, db_path, paths::database_path, paths, database_path}, + last_selection_path: get! {@path config, paths::last_selection_path, paths, last_selection_path}, + }, + download: DownloadConfig { + max_cache_size: { + let bytes_str: String = + get! {download::max_cache_size, config, download, max_cache_size}; + let number: Bytes = bytes_str + .parse() + .context("Failed to parse max_cache_size")?; + number + }, + }, + }) + } +} diff --git a/crates/yt/src/config/mod.rs b/crates/yt/src/config/mod.rs new file mode 100644 index 0000000..a10f7c2 --- /dev/null +++ b/crates/yt/src/config/mod.rs @@ -0,0 +1,76 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +#![allow(clippy::module_name_repetitions)] + +use std::path::PathBuf; + +use bytes::Bytes; +use serde::Serialize; + +mod default; +mod definitions; +pub mod file_system; + +#[derive(Serialize, Debug)] +pub struct Config { + pub global: GlobalConfig, + pub select: SelectConfig, + pub watch: WatchConfig, + pub paths: PathsConfig, + pub download: DownloadConfig, + pub update: UpdateConfig, +} +// These structures could get non-copy fields in the future. + +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct GlobalConfig { + pub display_colors: bool, +} +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct UpdateConfig { + pub max_backlog: usize, +} +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct DownloadConfig { + pub max_cache_size: Bytes, +} +#[derive(Serialize, Debug)] +pub struct SelectConfig { + pub playback_speed: f64, + pub subtitle_langs: String, +} +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct WatchConfig { + pub local_displays_length: usize, +} +#[derive(Serialize, Debug)] +pub struct PathsConfig { + pub download_dir: PathBuf, + pub mpv_config_path: PathBuf, + pub mpv_input_path: PathBuf, + pub database_path: PathBuf, + pub last_selection_path: PathBuf, +} + +// pub fn status_path() -> anyhow::Result<PathBuf> { +// const STATUS_PATH: &str = "running.info.json"; +// get_runtime_path(STATUS_PATH) +// } + +// pub fn subscriptions() -> anyhow::Result<PathBuf> { +// const SUBSCRIPTIONS: &str = "subscriptions.json"; +// get_data_path(SUBSCRIPTIONS) +// } diff --git a/crates/yt/src/constants.rs b/crates/yt/src/constants.rs new file mode 100644 index 0000000..0f5b918 --- /dev/null +++ b/crates/yt/src/constants.rs @@ -0,0 +1,12 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub const HELP_STR: &str = include_str!("./select/selection_file/help.str"); diff --git a/crates/yt/src/download/download_options.rs b/crates/yt/src/download/download_options.rs new file mode 100644 index 0000000..03c20ba --- /dev/null +++ b/crates/yt/src/download/download_options.rs @@ -0,0 +1,118 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Context; +use serde_json::{Value, json}; +use yt_dlp::{YoutubeDL, YoutubeDLOptions}; + +use crate::{app::App, storage::video_database::YtDlpOptions}; + +use super::progress_hook::wrapped_progress_hook; + +pub fn download_opts(app: &App, additional_opts: &YtDlpOptions) -> anyhow::Result<YoutubeDL> { + YoutubeDLOptions::new() + .with_progress_hook(wrapped_progress_hook) + .set("extract_flat", "in_playlist") + .set( + "extractor_args", + json! { + { + "youtube": { + "comment_sort": [ "top" ], + "max_comments": [ "150", "all", "100" ] + } + } + }, + ) + //.set("cookiesfrombrowser", json! {("firefox", "me.google", None::<String>, "youtube_dlp")}) + .set("prefer_free_formats", true) + .set("ffmpeg_location", env!("FFMPEG_LOCATION")) + .set("format", "bestvideo[height<=?1080]+bestaudio/best") + .set("fragment_retries", 10) + .set("getcomments", true) + .set("ignoreerrors", false) + .set("retries", 10) + .set("writeinfojson", true) + // NOTE: This results in a constant warning message. <2025-01-04> + //.set("writeannotations", true) + .set("writesubtitles", true) + .set("writeautomaticsub", true) + .set( + "outtmpl", + json! { + { + "default": app.config.paths.download_dir.join("%(channel)s/%(title)s.%(ext)s"), + "chapter": "%(title)s - %(section_number)03d %(section_title)s [%(id)s].%(ext)s" + } + }, + ) + .set("compat_opts", json! {{}}) + .set("forceprint", json! {{}}) + .set("print_to_file", json! {{}}) + .set("windowsfilenames", false) + .set("restrictfilenames", false) + .set("trim_file_names", false) + .set( + "postprocessors", + json! { + [ + { + "api": "https://sponsor.ajay.app", + "categories": [ + "interaction", + "intro", + "music_offtopic", + "sponsor", + "outro", + "poi_highlight", + "preview", + "selfpromo", + "filler", + "chapter" + ], + "key": "SponsorBlock", + "when": "after_filter" + }, + { + "force_keyframes": false, + "key": "ModifyChapters", + "remove_chapters_patterns": [], + "remove_ranges": [], + "remove_sponsor_segments": [ "sponsor" ], + "sponsorblock_chapter_title": "[SponsorBlock]: %(category_names)l" + }, + { + "add_chapters": true, + "add_infojson": null, + "add_metadata": false, + "key": "FFmpegMetadata" + }, + { + "key": "FFmpegConcat", + "only_multi_video": true, + "when": "playlist" + } + ] + }, + ) + .set( + "subtitleslangs", + Value::Array( + additional_opts + .subtitle_langs + .split(',') + .map(|val| Value::String(val.to_owned())) + .collect::<Vec<_>>(), + ), + ) + .build() + .context("Failed to instanciate download yt_dlp") +} diff --git a/crates/yt/src/download/mod.rs b/crates/yt/src/download/mod.rs new file mode 100644 index 0000000..110bf55 --- /dev/null +++ b/crates/yt/src/download/mod.rs @@ -0,0 +1,366 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, io, str::FromStr, sync::Arc, time::Duration}; + +use crate::{ + app::App, + download::download_options::download_opts, + storage::video_database::{ + Video, YtDlpOptions, + downloader::{get_next_uncached_video, set_video_cache_path}, + extractor_hash::ExtractorHash, + get::get_video_yt_dlp_opts, + notify::wait_for_cache_reduction, + }, + unreachable::Unreachable, +}; + +use anyhow::{Context, Result, bail}; +use bytes::Bytes; +use futures::{FutureExt, future::BoxFuture}; +use log::{debug, error, info, warn}; +use tokio::{fs, task::JoinHandle, time}; +use yt_dlp::{json_cast, json_get}; + +#[allow(clippy::module_name_repetitions)] +pub mod download_options; +pub mod progress_hook; + +#[derive(Debug)] +#[allow(clippy::module_name_repetitions)] +pub struct CurrentDownload { + task_handle: JoinHandle<Result<()>>, + extractor_hash: ExtractorHash, +} + +impl CurrentDownload { + fn new_from_video(app: Arc<App>, video: Video) -> Self { + let extractor_hash = video.extractor_hash; + + let task_handle = tokio::spawn(async move { + Downloader::actually_cache_video(&app, &video) + .await + .with_context(|| format!("Failed to cache video: '{}'", video.title))?; + Ok(()) + }); + + Self { + task_handle, + extractor_hash, + } + } +} + +enum CacheSizeCheck { + /// The video can be downloaded + Fits, + + /// The video and the current cache size together would exceed the size + TooLarge, + + /// The video would not even fit into the empty cache + ExceedsMaxCacheSize, +} + +#[derive(Debug)] +pub struct Downloader { + current_download: Option<CurrentDownload>, + video_size_cache: HashMap<ExtractorHash, u64>, + printed_warning: bool, + cached_cache_allocation: Option<Bytes>, +} + +impl Default for Downloader { + fn default() -> Self { + Self::new() + } +} + +impl Downloader { + #[must_use] + pub fn new() -> Self { + Self { + current_download: None, + video_size_cache: HashMap::new(), + printed_warning: false, + cached_cache_allocation: None, + } + } + + /// Check if enough cache is available. Will wait for 10s if it's not. + async fn is_enough_cache_available( + &mut self, + app: &App, + max_cache_size: u64, + next_video: &Video, + ) -> Result<CacheSizeCheck> { + if let Some(cdownload) = &self.current_download { + if cdownload.extractor_hash == next_video.extractor_hash { + // If the video is already being downloaded it will always fit. Otherwise the + // download would not have been started. + return Ok(CacheSizeCheck::Fits); + } + } + let cache_allocation = Self::get_current_cache_allocation(app).await?; + let video_size = self.get_approx_video_size(app, next_video)?; + + if video_size >= max_cache_size { + error!( + "The video '{}' ({}) exceeds the maximum cache size ({})! \ + Please set a bigger maximum (`--max-cache-size`) or skip it.", + next_video.title, + Bytes::new(video_size), + Bytes::new(max_cache_size) + ); + + return Ok(CacheSizeCheck::ExceedsMaxCacheSize); + } + + if cache_allocation.as_u64() + video_size >= max_cache_size { + if !self.printed_warning { + warn!( + "Can't download video: '{}' ({}) as it's too large for the cache ({} of {} allocated). \ + Waiting for cache size reduction..", + next_video.title, + Bytes::new(video_size), + &cache_allocation, + Bytes::new(max_cache_size) + ); + self.printed_warning = true; + + // Update this value immediately. + // This avoids printing the "Current cache size has changed .." warning below. + self.cached_cache_allocation = Some(cache_allocation); + } + + if let Some(cca) = self.cached_cache_allocation { + if cca != cache_allocation { + // Only print the warning if the display string has actually changed. + // Otherwise, we might confuse the user + if cca.to_string() != cache_allocation.to_string() { + warn!( + "Current cache size has changed, it's now: '{}'", + cache_allocation + ); + } + debug!( + "Cache size has changed: {} -> {}", + cca.as_u64(), + cache_allocation.as_u64() + ); + self.cached_cache_allocation = Some(cache_allocation); + } + } else { + unreachable!( + "The `printed_warning` should be false in this case, \ + and thus should have already set the `cached_cache_allocation`." + ); + } + + // Wait and hope, that a large video is deleted from the cache. + wait_for_cache_reduction(app).await?; + Ok(CacheSizeCheck::TooLarge) + } else { + self.printed_warning = false; + Ok(CacheSizeCheck::Fits) + } + } + + /// The entry point to the Downloader. + /// This Downloader will periodically check if the database has changed, and then also + /// change which videos it downloads. + /// This will run, until the database doesn't contain any watchable videos + pub async fn consume(&mut self, app: Arc<App>, max_cache_size: u64) -> Result<()> { + while let Some(next_video) = get_next_uncached_video(&app).await? { + match self + .is_enough_cache_available(&app, max_cache_size, &next_video) + .await? + { + CacheSizeCheck::Fits => (), + CacheSizeCheck::TooLarge => continue, + CacheSizeCheck::ExceedsMaxCacheSize => bail!("Giving up."), + }; + + if self.current_download.is_some() { + let current_download = self.current_download.take().unreachable("It is `Some`."); + + if current_download.task_handle.is_finished() { + current_download.task_handle.await??; + continue; + } + + if next_video.extractor_hash == current_download.extractor_hash { + // Reset the taken value + self.current_download = Some(current_download); + } else { + info!( + "Noticed, that the next video is not the video being downloaded, replacing it ('{}' vs. '{}')!", + next_video.extractor_hash.into_short_hash(&app).await?, + current_download + .extractor_hash + .into_short_hash(&app) + .await? + ); + + // Replace the currently downloading video + // FIXME(@bpeetz): This does not work (probably because of the python part.) <2025-02-21> + current_download.task_handle.abort(); + + let new_current_download = + CurrentDownload::new_from_video(Arc::clone(&app), next_video); + + self.current_download = Some(new_current_download); + } + } else { + info!( + "No video is being downloaded right now, setting it to '{}'", + next_video.title + ); + let new_current_download = + CurrentDownload::new_from_video(Arc::clone(&app), next_video); + self.current_download = Some(new_current_download); + } + + // TODO(@bpeetz): Why do we sleep here? <2025-02-21> + time::sleep(Duration::from_secs(1)).await; + } + + info!("Finished downloading!"); + Ok(()) + } + + pub async fn get_current_cache_allocation(app: &App) -> Result<Bytes> { + fn dir_size(mut dir: fs::ReadDir) -> BoxFuture<'static, Result<Bytes>> { + async move { + let mut acc = 0; + while let Some(entry) = dir.next_entry().await? { + let size = match entry.metadata().await? { + data if data.is_dir() => { + let path = entry.path(); + let read_dir = fs::read_dir(path).await?; + + dir_size(read_dir).await?.as_u64() + } + data => data.len(), + }; + acc += size; + } + Ok(Bytes::new(acc)) + } + .boxed() + } + + let read_dir_result = match fs::read_dir(&app.config.paths.download_dir).await { + Ok(ok) => ok, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => { + fs::create_dir_all(&app.config.paths.download_dir) + .await + .with_context(|| { + format!( + "Failed to create download dir at: '{}'", + &app.config.paths.download_dir.display() + ) + })?; + + info!( + "Created empty download dir at '{}'", + &app.config.paths.download_dir.display(), + ); + + // The new dir should not contain anything (otherwise we would not have had to + // create it) + return Ok(Bytes::new(0)); + } + err => Err(io::Error::from(err)).with_context(|| { + format!( + "Failed to get dir size of download dir at: '{}'", + &app.config.paths.download_dir.display() + ) + })?, + }, + }; + + dir_size(read_dir_result).await + } + + fn get_approx_video_size(&mut self, app: &App, video: &Video) -> Result<u64> { + if let Some(value) = self.video_size_cache.get(&video.extractor_hash) { + Ok(*value) + } else { + // the subtitle file size should be negligible + let add_opts = YtDlpOptions { + subtitle_langs: String::new(), + }; + let yt_dlp = download_opts(app, &add_opts)?; + + let result = yt_dlp + .extract_info(&video.url, false, true) + .with_context(|| { + format!("Failed to extract video information: '{}'", video.title) + })?; + + let size = if let Some(val) = result.get("filesize") { + json_cast!(val, as_u64) + } else if let Some(val) = result.get("filesize_approx") { + json_cast!(val, as_u64) + } else if result.get("duration").is_some() && result.get("tbr").is_some() { + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let duration = json_get!(result, "duration", as_f64).ceil() as u64; + + // TODO: yt_dlp gets this from the format + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let tbr = json_get!(result, "tbr", as_f64).ceil() as u64; + + duration * tbr * (1000 / 8) + } else { + let hardcoded_default = Bytes::from_str("250 MiB").expect("This is hardcoded"); + error!( + "Failed to find a filesize for video: '{}' (Using hardcoded value of {})", + video.title, hardcoded_default + ); + hardcoded_default.as_u64() + }; + + assert_eq!( + self.video_size_cache.insert(video.extractor_hash, size), + None + ); + + Ok(size) + } + } + + async fn actually_cache_video(app: &App, video: &Video) -> Result<()> { + debug!("Download started: {}", &video.title); + + let addional_opts = get_video_yt_dlp_opts(app, &video.extractor_hash).await?; + let yt_dlp = download_opts(app, &addional_opts)?; + + let result = yt_dlp + .download(&[video.url.to_owned()]) + .with_context(|| format!("Failed to download video: '{}'", video.title))?; + + assert_eq!(result.len(), 1); + let result = &result[0]; + + set_video_cache_path(app, &video.extractor_hash, Some(result)).await?; + + info!( + "Video '{}' was downlaoded to path: {}", + video.title, + result.display() + ); + + Ok(()) + } +} diff --git a/crates/yt/src/download/progress_hook.rs b/crates/yt/src/download/progress_hook.rs new file mode 100644 index 0000000..b75ec00 --- /dev/null +++ b/crates/yt/src/download/progress_hook.rs @@ -0,0 +1,188 @@ +use std::{ + io::{Write, stderr}, + process, +}; + +use bytes::Bytes; +use log::{Level, log_enabled}; +use yt_dlp::mk_python_function; + +use crate::{ + ansi_escape_codes::{clear_whole_line, move_to_col}, + select::selection_file::duration::MaybeDuration, +}; + +/// # Panics +/// If expectations fail. +#[allow(clippy::too_many_lines, clippy::needless_pass_by_value)] +pub fn progress_hook( + input: serde_json::Map<String, serde_json::Value>, +) -> Result<(), std::io::Error> { + // Only add the handler, if the log-level is higher than Debug (this avoids covering debug + // messages). + if log_enabled!(Level::Debug) { + return Ok(()); + } + + macro_rules! get { + (@interrogate $item:ident, $type_fun:ident, $get_fun:ident, $name:expr) => {{ + let a = $item.get($name).expect(concat!( + "The field '", + stringify!($name), + "' should exist." + )); + + if a.$type_fun() { + a.$get_fun().expect( + "The should have been checked in the if guard, so unpacking here is fine", + ) + } else { + panic!( + "Value {} => \n{}\n is not of type: {}", + $name, + a, + stringify!($type_fun) + ); + } + }}; + + ($type_fun:ident, $get_fun:ident, $name1:expr, $name2:expr) => {{ + let a = get! {@interrogate input, is_object, as_object, $name1}; + let b = get! {@interrogate a, $type_fun, $get_fun, $name2}; + b + }}; + + ($type_fun:ident, $get_fun:ident, $name:expr) => {{ + get! {@interrogate input, $type_fun, $get_fun, $name} + }}; + } + + macro_rules! default_get { + (@interrogate $item:ident, $default:expr, $get_fun:ident, $name:expr) => {{ + let a = if let Some(field) = $item.get($name) { + field.$get_fun().unwrap_or($default) + } else { + $default + }; + a + }}; + + ($get_fun:ident, $default:expr, $name1:expr, $name2:expr) => {{ + let a = get! {@interrogate input, is_object, as_object, $name1}; + let b = default_get! {@interrogate a, $default, $get_fun, $name2}; + b + }}; + + ($get_fun:ident, $default:expr, $name:expr) => {{ + default_get! {@interrogate input, $default, $get_fun, $name} + }}; + } + + macro_rules! c { + ($color:expr, $format:expr) => { + format!("\x1b[{}m{}\x1b[0m", $color, $format) + }; + } + + #[allow(clippy::items_after_statements)] + fn format_bytes(bytes: u64) -> String { + let bytes = Bytes::new(bytes); + bytes.to_string() + } + + #[allow(clippy::items_after_statements)] + fn format_speed(speed: f64) -> String { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let bytes = Bytes::new(speed.floor() as u64); + format!("{bytes}/s") + } + + let get_title = || -> String { + match get! {is_string, as_str, "info_dict", "ext"} { + "vtt" => { + format!( + "Subtitles ({})", + default_get! {as_str, "<No Subtitle Language>", "info_dict", "name"} + ) + } + "webm" | "mp4" | "mp3" | "m4a" => { + default_get! { as_str, "<No title>", "info_dict", "title"}.to_owned() + } + other => panic!("The extension '{other}' is not yet implemented"), + } + }; + + match get! {is_string, as_str, "status"} { + "downloading" => { + let elapsed = default_get! {as_f64, 0.0f64, "elapsed"}; + let eta = default_get! {as_f64, 0.0, "eta"}; + let speed = default_get! {as_f64, 0.0, "speed"}; + + let downloaded_bytes = get! {is_u64, as_u64, "downloaded_bytes"}; + let (total_bytes, bytes_is_estimate): (u64, &'static str) = { + let total_bytes = default_get!(as_u64, 0, "total_bytes"); + if total_bytes == 0 { + let maybe_estimate = default_get!(as_u64, 0, "total_bytes_estimate"); + + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + if maybe_estimate == 0 { + // The download speed should be in bytes per second and the eta in seconds. + // Thus multiplying them gets us the raw bytes (which were estimated by `yt_dlp`, from their `info.json`) + let bytes_still_needed = (speed * eta).ceil() as u64; + + (downloaded_bytes + bytes_still_needed, "~") + } else { + (maybe_estimate, "~") + } + } else { + (total_bytes, "") + } + }; + + let percent: f64 = { + if total_bytes == 0 { + 100.0 + } else { + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + { + (downloaded_bytes as f64 / total_bytes as f64) * 100.0 + } + } + }; + + clear_whole_line(); + move_to_col(1); + + eprint!( + "'{}' [{}/{} at {}] -> [{} of {}{} {}] ", + c!("34;1", get_title()), + c!("33;1", MaybeDuration::from_secs_f64(elapsed)), + c!("33;1", MaybeDuration::from_secs_f64(eta)), + c!("32;1", format_speed(speed)), + c!("31;1", format_bytes(downloaded_bytes)), + c!("31;1", bytes_is_estimate), + c!("31;1", format_bytes(total_bytes)), + c!("36;1", format!("{:.02}%", percent)) + ); + stderr().flush()?; + } + "finished" => { + eprintln!("-> Finished downloading."); + } + "error" => { + // TODO: This should probably return an Err. But I'm not so sure where the error would + // bubble up to (i.e., who would catch it) <2025-01-21> + eprintln!("-> Error while downloading: {}", get_title()); + process::exit(1); + } + other => unreachable!("'{other}' should not be a valid state!"), + } + + Ok(()) +} + +mk_python_function!(progress_hook, wrapped_progress_hook); diff --git a/crates/yt/src/main.rs b/crates/yt/src/main.rs new file mode 100644 index 0000000..39f52f4 --- /dev/null +++ b/crates/yt/src/main.rs @@ -0,0 +1,247 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// `yt` is not a library. Besides, the `anyhow::Result` type is really useless, if you're not going +// to print it anyways. +#![allow(clippy::missing_errors_doc)] + +use std::sync::Arc; + +use anyhow::{Context, Result, bail}; +use app::App; +use bytes::Bytes; +use cache::{invalidate, maintain}; +use clap::Parser; +use cli::{CacheCommand, SelectCommand, SubscriptionCommand, VideosCommand}; +use config::Config; +use log::{error, info}; +use select::cmds::handle_select_cmd; +use storage::video_database::get::video_by_hash; +use tokio::{ + fs::File, + io::{BufReader, stdin}, + task::JoinHandle, +}; + +use crate::{cli::Command, storage::subscriptions}; + +pub mod ansi_escape_codes; +pub mod app; +pub mod cli; +pub mod unreachable; + +pub mod cache; +pub mod comments; +pub mod config; +pub mod constants; +pub mod download; +pub mod select; +pub mod status; +pub mod storage; +pub mod subscribe; +pub mod update; +pub mod version; +pub mod videos; +pub mod watch; + +#[tokio::main] +// This is _the_ main function after all. It is not really good, but it sort of works. +#[allow(clippy::too_many_lines)] +async fn main() -> Result<()> { + let args = cli::CliArgs::parse(); + + // The default verbosity is 1 (Warn) + let verbosity: u8 = args.verbosity + 1; + + stderrlog::new() + .module(module_path!()) + .modules(&["yt_dlp".to_owned(), "libmpv2".to_owned()]) + .quiet(args.quiet) + .show_module_names(false) + .color(stderrlog::ColorChoice::Auto) + .verbosity(verbosity as usize) + .timestamp(stderrlog::Timestamp::Off) + .init() + .expect("Let's just hope that this does not panic"); + + info!("Using verbosity level: '{} ({})'", verbosity, { + match verbosity { + 0 => "Error", + 1 => "Warn", + 2 => "Info", + 3 => "Debug", + 4.. => "Trace", + } + }); + + let config = Config::from_config_file(args.db_path, args.config_path, args.color)?; + if args.version { + version::show(&config).await?; + return Ok(()); + } + + let app = App::new(config, !args.no_migrate_db).await?; + + match args.command.unwrap_or(Command::default()) { + Command::Download { + force, + max_cache_size, + } => { + let max_cache_size = + max_cache_size.unwrap_or(app.config.download.max_cache_size.as_u64()); + info!("Max cache size: '{}'", Bytes::new(max_cache_size)); + + maintain(&app, false).await?; + if force { + invalidate(&app, true).await?; + } + + download::Downloader::new() + .consume(Arc::new(app), max_cache_size) + .await?; + } + Command::Select { cmd } => { + let cmd = cmd.unwrap_or(SelectCommand::default()); + + match cmd { + SelectCommand::File { + done, + use_last_selection, + } => Box::pin(select::select(&app, done, use_last_selection)).await?, + _ => Box::pin(handle_select_cmd(&app, cmd, None)).await?, + } + } + Command::Sedowa {} => { + Box::pin(select::select(&app, false, false)).await?; + + let arc_app = Arc::new(app); + dowa(arc_app).await?; + } + Command::Dowa {} => { + let arc_app = Arc::new(app); + dowa(arc_app).await?; + } + Command::Videos { cmd } => match cmd { + VideosCommand::List { + search_query, + limit, + } => { + videos::query(&app, limit, search_query) + .await + .context("Failed to query videos")?; + } + VideosCommand::Info { hash } => { + let video = video_by_hash(&app, &hash.realize(&app).await?).await?; + + print!( + "{}", + &video + .to_info_display(&app) + .await + .context("Failed to format video")? + ); + } + }, + Command::Update { + max_backlog, + subscriptions, + } => { + let all_subs = subscriptions::get(&app).await?; + + for sub in &subscriptions { + if !all_subs.0.contains_key(sub) { + bail!( + "Your specified subscription to update '{}' is not a subscription!", + sub + ) + } + } + + let max_backlog = max_backlog.unwrap_or(app.config.update.max_backlog); + + update::update(&app, max_backlog, subscriptions).await?; + } + Command::Subscriptions { cmd } => match cmd { + SubscriptionCommand::Add { name, url } => { + subscribe::subscribe(&app, name, url) + .await + .context("Failed to add a subscription")?; + } + SubscriptionCommand::Remove { name } => { + subscribe::unsubscribe(&app, name) + .await + .context("Failed to remove a subscription")?; + } + SubscriptionCommand::List {} => { + let all_subs = subscriptions::get(&app).await?; + + for (key, val) in all_subs.0 { + println!("{}: '{}'", key, val.url); + } + } + SubscriptionCommand::Export {} => { + let all_subs = subscriptions::get(&app).await?; + for val in all_subs.0.values() { + println!("{}", val.url); + } + } + SubscriptionCommand::Import { file, force } => { + if let Some(file) = file { + let f = File::open(file).await?; + + subscribe::import(&app, BufReader::new(f), force).await?; + } else { + subscribe::import(&app, BufReader::new(stdin()), force).await?; + } + } + }, + + Command::Watch {} => watch::watch(Arc::new(app)).await?, + Command::Playlist { watch } => watch::playlist::playlist(&app, watch).await?, + + Command::Status {} => status::show(&app).await?, + Command::Config {} => status::config(&app)?, + + Command::Database { command } => match command { + CacheCommand::Invalidate { hard } => invalidate(&app, hard).await?, + CacheCommand::Maintain { all } => maintain(&app, all).await?, + }, + + Command::Comments {} => { + comments::comments(&app).await?; + } + Command::Description {} => { + comments::description(&app).await?; + } + } + + Ok(()) +} + +async fn dowa(arc_app: Arc<App>) -> Result<()> { + let max_cache_size = arc_app.config.download.max_cache_size; + info!("Max cache size: '{}'", max_cache_size); + + let arc_app_clone = Arc::clone(&arc_app); + let download: JoinHandle<()> = tokio::spawn(async move { + let result = download::Downloader::new() + .consume(arc_app_clone, max_cache_size.as_u64()) + .await; + + if let Err(err) = result { + error!("Error from downloader: {err:?}"); + } + }); + + watch::watch(arc_app).await?; + download.await?; + Ok(()) +} diff --git a/crates/yt/src/select/cmds/add.rs b/crates/yt/src/select/cmds/add.rs new file mode 100644 index 0000000..387b3a1 --- /dev/null +++ b/crates/yt/src/select/cmds/add.rs @@ -0,0 +1,191 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + download::download_options::download_opts, + storage::video_database::{ + self, extractor_hash::ExtractorHash, get::get_all_hashes, set::add_video, + }, + update::video_entry_to_video, +}; + +use anyhow::{Context, Result, bail}; +use log::{error, warn}; +use url::Url; +use yt_dlp::{InfoJson, YoutubeDL, json_cast, json_get}; + +#[allow(clippy::too_many_lines)] +pub(super) async fn add( + app: &App, + urls: Vec<Url>, + start: Option<usize>, + stop: Option<usize>, +) -> Result<()> { + for url in urls { + async fn process_and_add(app: &App, entry: InfoJson, yt_dlp: &YoutubeDL) -> Result<()> { + let url = json_get!(entry, "url", as_str).parse()?; + + let entry = yt_dlp + .extract_info(&url, false, true) + .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; + + add_entry(app, entry).await?; + + Ok(()) + } + + async fn add_entry(app: &App, entry: InfoJson) -> Result<()> { + // We have to re-fetch all hashes every time, because a user could try to add the same + // URL twice (for whatever reason.) + let hashes = get_all_hashes(app) + .await + .context("Failed to fetch all video hashes")?; + let extractor_hash = blake3::hash(json_get!(entry, "id", as_str).as_bytes()); + if hashes.contains(&extractor_hash) { + error!( + "Video '{}'{} is already in the database. Skipped adding it", + ExtractorHash::from_hash(extractor_hash) + .into_short_hash(app) + .await + .with_context(|| format!( + "Failed to format hash of video '{}' as short hash", + entry + .get("url") + .map_or("<Unknown video Url>".to_owned(), ToString::to_string) + ))?, + entry + .get("title") + .map_or(String::new(), |title| format!(" ('{title}')")) + ); + return Ok(()); + } + + let video = video_entry_to_video(&entry, None)?; + add_video(app, video.clone()).await?; + + println!("{}", &video.to_line_display(app).await?); + + Ok(()) + } + + let yt_dlp = download_opts( + app, + &video_database::YtDlpOptions { + subtitle_langs: String::new(), + }, + )?; + + let entry = yt_dlp + .extract_info(&url, false, true) + .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; + + match entry.get("_type").map(|val| json_cast!(val, as_str)) { + Some("Video") => { + add_entry(app, entry).await?; + if start.is_some() || stop.is_some() { + warn!( + "You added `start` and/or `stop` markers for a single *video*! These will be ignored." + ); + } + } + Some("Playlist") => { + if let Some(entries) = entry.get("entries") { + let entries = json_cast!(entries, as_array); + let start = start.unwrap_or(0); + let stop = stop.unwrap_or(entries.len() - 1); + + let respected_entries = + take_vector(entries, start, stop).with_context(|| { + format!( + "Failed to take entries starting at: {start} and ending with {stop}" + ) + })?; + + if respected_entries.is_empty() { + warn!("No entries found, after applying your start/stop limits."); + } else { + // Pre-warm the cache + process_and_add( + app, + json_cast!(respected_entries[0], as_object).to_owned(), + &yt_dlp, + ) + .await?; + let respected_entries = &respected_entries[1..]; + + let futures: Vec<_> = respected_entries + .iter() + .map(|entry| { + process_and_add( + app, + json_cast!(entry, as_object).to_owned(), + &yt_dlp, + ) + }) + .collect(); + + for fut in futures { + fut.await?; + } + } + } else { + bail!("Your playlist does not seem to have any entries!") + } + } + other => bail!( + "Your URL should point to a video or a playlist, but points to a '{:#?}'", + other + ), + } + } + + Ok(()) +} + +fn take_vector<T>(vector: &[T], start: usize, stop: usize) -> Result<&[T]> { + let length = vector.len(); + + if stop >= length { + bail!( + "Your stop marker ({stop}) exceeds the possible entries ({length})! Remember that it is zero indexed." + ); + } + + Ok(&vector[start..=stop]) +} + +#[cfg(test)] +mod test { + use crate::select::cmds::add::take_vector; + + #[test] + fn test_vector_take() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + let new_vec = take_vector(&vec, 2, 8).unwrap(); + + assert_eq!(new_vec, vec![2, 3, 4, 5, 6, 7, 8]); + } + + #[test] + fn test_vector_take_overflow() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + assert!(take_vector(&vec, 0, 12).is_err()); + } + + #[test] + fn test_vector_take_equal() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + assert!(take_vector(&vec, 0, 11).is_err()); + } +} diff --git a/crates/yt/src/select/cmds/mod.rs b/crates/yt/src/select/cmds/mod.rs new file mode 100644 index 0000000..aabcd3d --- /dev/null +++ b/crates/yt/src/select/cmds/mod.rs @@ -0,0 +1,111 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use crate::{ + app::App, + cli::{SelectCommand, SharedSelectionCommandArgs}, + storage::video_database::{ + Priority, VideoOptions, VideoStatus, + get::video_by_hash, + set::{set_video_options, video_status}, + }, +}; + +use anyhow::{Context, Result, bail}; + +mod add; + +pub async fn handle_select_cmd( + app: &App, + cmd: SelectCommand, + line_number: Option<i64>, +) -> Result<()> { + match cmd { + SelectCommand::Pick { shared } => { + handle_status_change(app, shared, line_number, VideoStatus::Pick).await?; + } + SelectCommand::Drop { shared } => { + handle_status_change(app, shared, line_number, VideoStatus::Drop).await?; + } + SelectCommand::Watched { shared } => { + handle_status_change(app, shared, line_number, VideoStatus::Watched).await?; + } + SelectCommand::Add { urls, start, stop } => { + Box::pin(add::add(app, urls, start, stop)).await?; + } + SelectCommand::Watch { shared } => { + let hash = shared.hash.clone().realize(app).await?; + + let video = video_by_hash(app, &hash).await?; + + if let VideoStatus::Cached { + cache_path, + is_focused, + } = video.status + { + handle_status_change( + app, + shared, + line_number, + VideoStatus::Cached { + cache_path, + is_focused, + }, + ) + .await?; + } else { + handle_status_change(app, shared, line_number, VideoStatus::Watch).await?; + } + } + + SelectCommand::Url { shared } => { + let Some(url) = shared.url else { + bail!("You need to provide a url to `select url ..`") + }; + + let mut firefox = std::process::Command::new("firefox"); + firefox.args(["-P", "timesinks.youtube"]); + firefox.arg(url.as_str()); + let _handle = firefox.spawn().context("Failed to run firefox")?; + } + SelectCommand::File { .. } => unreachable!("This should have been filtered out"), + } + Ok(()) +} + +async fn handle_status_change( + app: &App, + shared: SharedSelectionCommandArgs, + line_number: Option<i64>, + new_status: VideoStatus, +) -> Result<()> { + let hash = shared.hash.realize(app).await?; + let video_options = VideoOptions::new( + shared + .subtitle_langs + .unwrap_or(app.config.select.subtitle_langs.clone()), + shared.speed.unwrap_or(app.config.select.playback_speed), + ); + let priority = compute_priority(line_number, shared.priority); + + video_status(app, &hash, new_status, priority).await?; + set_video_options(app, &hash, &video_options).await?; + + Ok(()) +} + +fn compute_priority(line_number: Option<i64>, priority: Option<i64>) -> Option<Priority> { + if let Some(pri) = priority { + Some(Priority::from(pri)) + } else { + line_number.map(Priority::from) + } +} diff --git a/crates/yt/src/select/mod.rs b/crates/yt/src/select/mod.rs new file mode 100644 index 0000000..8db9ae3 --- /dev/null +++ b/crates/yt/src/select/mod.rs @@ -0,0 +1,176 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + env::{self}, + fs, + io::{BufRead, BufReader, BufWriter, Write}, + string::String, +}; + +use crate::{ + app::App, + cli::CliArgs, + constants::HELP_STR, + storage::video_database::{Video, VideoStatusMarker, get}, + unreachable::Unreachable, +}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use cmds::handle_select_cmd; +use futures::{TryStreamExt, stream::FuturesOrdered}; +use selection_file::process_line; +use tempfile::Builder; +use tokio::process::Command; + +pub mod cmds; +pub mod selection_file; + +async fn to_select_file_display_owned(video: Video, app: &App) -> Result<String> { + video.to_select_file_display(app).await +} + +pub async fn select(app: &App, done: bool, use_last_selection: bool) -> Result<()> { + let temp_file = Builder::new() + .prefix("yt_video_select-") + .suffix(".yts") + .rand_bytes(6) + .tempfile() + .context("Failed to get tempfile")?; + + if use_last_selection { + fs::copy(&app.config.paths.last_selection_path, &temp_file)?; + } else { + let matching_videos = if done { + get::videos(app, VideoStatusMarker::ALL).await? + } else { + get::videos( + app, + &[ + VideoStatusMarker::Pick, + // + VideoStatusMarker::Watch, + VideoStatusMarker::Cached, + ], + ) + .await? + }; + + // Warmup the cache for the display rendering of the videos. + // Otherwise the futures would all try to warm it up at the same time. + if let Some(vid) = matching_videos.first() { + drop(vid.to_line_display(app).await?); + } + + let mut edit_file = BufWriter::new(&temp_file); + + matching_videos + .into_iter() + .map(|vid| to_select_file_display_owned(vid, app)) + .collect::<FuturesOrdered<_>>() + .try_collect::<Vec<String>>() + .await? + .into_iter() + .try_for_each(|line| -> Result<()> { + edit_file + .write_all(line.as_bytes()) + .context("Failed to write to `edit_file`")?; + + Ok(()) + })?; + + edit_file.write_all(HELP_STR.as_bytes())?; + edit_file.flush().context("Failed to flush edit file")?; + }; + + { + let editor = env::var("EDITOR").unwrap_or("nvim".to_owned()); + + let mut nvim = Command::new(editor); + nvim.arg(temp_file.path()); + let status = nvim.status().await.context("Falied to run nvim")?; + if !status.success() { + bail!("nvim exited with error status: {}", status) + } + } + + let read_file = temp_file.reopen()?; + fs::copy(temp_file.path(), &app.config.paths.last_selection_path) + .context("Failed to persist selection file")?; + + let reader = BufReader::new(&read_file); + + let mut line_number = 0; + for line in reader.lines() { + let line = line.context("Failed to read a line")?; + + if let Some(line) = process_line(&line)? { + line_number -= 1; + + // debug!( + // "Parsed command: `{}`", + // line.iter() + // .map(|val| format!("\"{}\"", val)) + // .collect::<Vec<String>>() + // .join(" ") + // ); + + let arg_line = ["yt", "select"] + .into_iter() + .chain(line.iter().map(String::as_str)); + + let args = CliArgs::parse_from(arg_line); + + let crate::cli::Command::Select { cmd } = args + .command + .unreachable("This will be some, as we constructed it above.") + else { + unreachable!("This is checked in the `filter_line` function") + }; + + Box::pin(handle_select_cmd( + app, + cmd.unreachable( + "This value should always be some \ + here, as it would otherwise thrown an error above.", + ), + Some(line_number), + )) + .await?; + } + } + + Ok(()) +} + +// // FIXME: There should be no reason why we need to re-run yt, just to get the help string. But I've +// // yet to find a way to do it without the extra exec <2024-08-20> +// async fn get_help() -> Result<String> { +// let binary_name = current_exe()?; +// let cmd = Command::new(binary_name) +// .args(&["select", "--help"]) +// .output() +// .await?; +// +// assert_eq!(cmd.status.code(), Some(0)); +// +// let output = String::from_utf8(cmd.stdout).expect("Our help output was not utf8?"); +// +// let out = output +// .lines() +// .map(|line| format!("# {}\n", line)) +// .collect::<String>(); +// +// debug!("Returning help: '{}'", &out); +// +// Ok(out) +// } diff --git a/crates/yt/src/select/selection_file/duration.rs b/crates/yt/src/select/selection_file/duration.rs new file mode 100644 index 0000000..77c4fc5 --- /dev/null +++ b/crates/yt/src/select/selection_file/duration.rs @@ -0,0 +1,185 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::str::FromStr; +use std::time::Duration; + +use anyhow::{Context, Result}; + +const SECOND: u64 = 1; +const MINUTE: u64 = 60 * SECOND; +const HOUR: u64 = 60 * MINUTE; +const DAY: u64 = 24 * HOUR; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct MaybeDuration { + time: Option<Duration>, +} + +impl MaybeDuration { + #[must_use] + pub fn from_std(d: Duration) -> Self { + Self { time: Some(d) } + } + + #[must_use] + pub fn from_secs_f64(d: f64) -> Self { + Self { + time: Some(Duration::from_secs_f64(d)), + } + } + #[must_use] + pub fn from_maybe_secs_f64(d: Option<f64>) -> Self { + Self { + time: d.map(Duration::from_secs_f64), + } + } + #[must_use] + pub fn from_secs(d: u64) -> Self { + Self { + time: Some(Duration::from_secs(d)), + } + } + + #[must_use] + pub fn zero() -> Self { + Self { + time: Some(Duration::default()), + } + } + + /// Try to return the current duration encoded as seconds. + #[must_use] + pub fn as_secs(&self) -> Option<u64> { + self.time.map(|v| v.as_secs()) + } + + /// Try to return the current duration encoded as seconds and nanoseconds. + #[must_use] + pub fn as_secs_f64(&self) -> Option<f64> { + self.time.map(|v| v.as_secs_f64()) + } +} + +impl FromStr for MaybeDuration { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + fn parse_num(str: &str, suffix: char) -> Result<u64> { + str.strip_suffix(suffix) + .with_context(|| format!("Failed to strip suffix '{suffix}' of number: '{str}'"))? + .parse::<u64>() + .with_context(|| format!("Failed to parse '{suffix}'")) + } + + if s == "[No duration]" { + return Ok(Self { time: None }); + } + + let buf: Vec<_> = s.split(' ').collect(); + + let days; + let hours; + let minutes; + let seconds; + + assert_eq!(buf.len(), 2, "Other lengths should not happen"); + + if buf[0].ends_with('d') { + days = parse_num(buf[0], 'd')?; + hours = parse_num(buf[1], 'h')?; + minutes = parse_num(buf[2], 'm')?; + seconds = 0; + } else if buf[0].ends_with('h') { + days = 0; + hours = parse_num(buf[0], 'h')?; + minutes = parse_num(buf[1], 'm')?; + seconds = 0; + } else if buf[0].ends_with('m') { + days = 0; + hours = 0; + minutes = parse_num(buf[0], 'm')?; + seconds = parse_num(buf[1], 's')?; + } else { + unreachable!( + "The first part always ends with 'h' or 'm', but was: {:#?}", + buf + ) + } + + Ok(Self { + time: Some(Duration::from_secs( + days * DAY + hours * HOUR + minutes * MINUTE + seconds * SECOND, + )), + }) + } +} + +impl std::fmt::Display for MaybeDuration { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + if let Some(self_seconds) = self.as_secs() { + let base_day = self_seconds - (self_seconds % DAY); + let base_hour = (self_seconds % DAY) - ((self_seconds % DAY) % HOUR); + let base_min = (self_seconds % HOUR) - (((self_seconds % DAY) % HOUR) % MINUTE); + let base_sec = ((self_seconds % DAY) % HOUR) % MINUTE; + + let d = base_day / DAY; + let h = base_hour / HOUR; + let m = base_min / MINUTE; + let s = base_sec / SECOND; + + if d > 0 { + write!(fmt, "{d}d {h}h {m}m") + } else if h > 0 { + write!(fmt, "{h}h {m}m") + } else { + write!(fmt, "{m}m {s}s") + } + } else { + write!(fmt, "[No duration]") + } + } +} +#[cfg(test)] +mod test { + use std::str::FromStr; + + use crate::select::selection_file::duration::{DAY, HOUR, MINUTE}; + + use super::MaybeDuration; + + #[test] + fn test_display_duration_1h() { + let dur = MaybeDuration::from_secs(HOUR); + assert_eq!("1h 0m".to_owned(), dur.to_string()); + } + #[test] + fn test_display_duration_30min() { + let dur = MaybeDuration::from_secs(MINUTE * 30); + assert_eq!("30m 0s".to_owned(), dur.to_string()); + } + #[test] + fn test_display_duration_1d() { + let dur = MaybeDuration::from_secs(DAY + MINUTE * 30 + HOUR * 2); + assert_eq!("1d 2h 30m".to_owned(), dur.to_string()); + } + + #[test] + fn test_display_duration_roundtrip() { + let dur = MaybeDuration::zero(); + let dur_str = dur.to_string(); + + assert_eq!( + MaybeDuration::zero(), + MaybeDuration::from_str(&dur_str).unwrap() + ); + } +} diff --git a/crates/yt/src/select/selection_file/help.str b/crates/yt/src/select/selection_file/help.str new file mode 100644 index 0000000..e3cc347 --- /dev/null +++ b/crates/yt/src/select/selection_file/help.str @@ -0,0 +1,12 @@ +# Commands: +# w, watch [-p,-s,-l] Mark the video given by the hash to be watched +# wd, watched [-p,-s,-l] Mark the video given by the hash as already watched +# d, drop [-p,-s,-l] Mark the video given by the hash to be dropped +# u, url [-p,-s,-l] Open the video URL in Firefox's `timesinks.youtube` profile +# p, pick [-p,-s,-l] Reset the videos status to 'Pick' +# a, add URL Add a video, defined by the URL +# +# See `yt select <cmd_name> --help` for more help. +# +# These lines can be re-ordered; they are executed from top to bottom. +# vim: filetype=yts conceallevel=2 concealcursor=nc colorcolumn= nowrap diff --git a/crates/yt/src/select/selection_file/help.str.license b/crates/yt/src/select/selection_file/help.str.license new file mode 100644 index 0000000..a0e196c --- /dev/null +++ b/crates/yt/src/select/selection_file/help.str.license @@ -0,0 +1,10 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +SPDX-License-Identifier: GPL-3.0-or-later + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/crates/yt/src/select/selection_file/mod.rs b/crates/yt/src/select/selection_file/mod.rs new file mode 100644 index 0000000..abd26c4 --- /dev/null +++ b/crates/yt/src/select/selection_file/mod.rs @@ -0,0 +1,32 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! The data structures needed to express the file, which the user edits + +use anyhow::{Context, Result}; +use trinitry::Trinitry; + +pub mod duration; + +pub fn process_line(line: &str) -> Result<Option<Vec<String>>> { + // Filter out comments and empty lines + if line.starts_with('#') || line.trim().is_empty() { + Ok(None) + } else { + let tri = Trinitry::new(line).with_context(|| format!("Failed to parse line '{line}'"))?; + + let mut vec = Vec::with_capacity(tri.arguments().len() + 1); + vec.push(tri.command().to_owned()); + vec.extend(tri.arguments().to_vec()); + + Ok(Some(vec)) + } +} diff --git a/crates/yt/src/status/mod.rs b/crates/yt/src/status/mod.rs new file mode 100644 index 0000000..18bef7d --- /dev/null +++ b/crates/yt/src/status/mod.rs @@ -0,0 +1,129 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::time::Duration; + +use crate::{ + app::App, + download::Downloader, + select::selection_file::duration::MaybeDuration, + storage::{ + subscriptions, + video_database::{VideoStatusMarker, get}, + }, +}; + +use anyhow::{Context, Result}; +use bytes::Bytes; + +macro_rules! get { + ($videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .count() + }; + + (@collect $videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .collect() + }; +} + +pub async fn show(app: &App) -> Result<()> { + let all_videos = get::videos(app, VideoStatusMarker::ALL).await?; + + // lengths + let picked_videos_len = get!(all_videos, Pick); + + let watch_videos_len = get!(all_videos, Watch); + let cached_videos_len = get!(all_videos, Cached); + let watched_videos_len = get!(all_videos, Watched); + let watched_videos: Vec<_> = get!(@collect all_videos, Watched); + + let drop_videos_len = get!(all_videos, Drop); + let dropped_videos_len = get!(all_videos, Dropped); + + let subscriptions = subscriptions::get(app).await?; + let subscriptions_len = subscriptions.0.len(); + + let watchtime_status = { + let total_watch_time_raw = watched_videos + .iter() + .fold(Duration::default(), |acc, vid| acc + vid.watch_progress); + + // Most things are watched at a speed of s (which is defined in the config file). + // Thus + // y = x * s -> y / s = x + let total_watch_time = Duration::from_secs_f64( + (total_watch_time_raw.as_secs_f64()) / app.config.select.playback_speed, + ); + + let speed = app.config.select.playback_speed; + + // Do not print the adjusted time, if the user has keep the speed level at 1. + #[allow(clippy::float_cmp)] + if speed == 1.0 { + format!( + "Total Watchtime: {}\n", + MaybeDuration::from_std(total_watch_time_raw) + ) + } else { + format!( + "Total Watchtime: {} (at {speed} speed: {})\n", + MaybeDuration::from_std(total_watch_time_raw), + MaybeDuration::from_std(total_watch_time), + ) + } + }; + + let watch_rate: f64 = { + fn to_f64(input: usize) -> f64 { + f64::from(u32::try_from(input).expect("This should never exceed u32::MAX")) + } + + let count = to_f64(watched_videos_len) / (to_f64(drop_videos_len) + to_f64(dropped_videos_len)); + count * 100.0 + }; + + let cache_usage_raw = Downloader::get_current_cache_allocation(app) + .await + .context("Failed to get current cache allocation")?; + let cache_usage: Bytes = cache_usage_raw; + println!( + "\ +Picked Videos: {picked_videos_len} + +Watch Videos: {watch_videos_len} +Cached Videos: {cached_videos_len} +Watched Videos: {watched_videos_len} (watch rate: {watch_rate:.2} %) + +Drop Videos: {drop_videos_len} +Dropped Videos: {dropped_videos_len} + +{watchtime_status} + + Subscriptions: {subscriptions_len} + Cache usage: {cache_usage}" + ); + + Ok(()) +} + +pub fn config(app: &App) -> Result<()> { + let config_str = toml::to_string(&app.config)?; + + print!("{config_str}"); + + Ok(()) +} diff --git a/crates/yt/src/storage/migrate/mod.rs b/crates/yt/src/storage/migrate/mod.rs new file mode 100644 index 0000000..953d079 --- /dev/null +++ b/crates/yt/src/storage/migrate/mod.rs @@ -0,0 +1,279 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fmt::Display, + future::Future, + time::{SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result, bail}; +use chrono::TimeDelta; +use log::{debug, info}; +use sqlx::{Sqlite, SqlitePool, Transaction, query}; + +use crate::app::App; + +macro_rules! make_upgrade { + ($app:expr, $old_version:expr, $new_version:expr, $sql_name:expr) => { + add_error_context( + async { + let mut tx = $app + .database + .begin() + .await + .context("Failed to start the update transaction")?; + debug!("Migrating: {} -> {}", $old_version, $new_version); + + sqlx::raw_sql(include_str!($sql_name)) + .execute(&mut *tx) + .await + .context("Failed to run the update sql script")?; + + set_db_version( + &mut tx, + if $old_version == Self::Empty { + // There is no previous version we would need to remove + None + } else { + Some($old_version) + }, + $new_version, + ) + .await + .with_context(|| format!("Failed to set the new version ({})", $new_version))?; + + tx.commit() + .await + .context("Failed to commit the update transaction")?; + + // NOTE: This is needed, so that sqlite "sees" our changes to the table + // without having to reconnect. <2025-02-18> + query!("VACUUM") + .execute(&$app.database) + .await + .context("Failed to vacuum database")?; + + Ok(()) + }, + $new_version, + ) + .await?; + + Box::pin($new_version.update($app)).await.context(concat!( + "While updating to version: ", + stringify!($new_version) + )) + }; +} + +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub enum DbVersion { + /// The database is not yet initialized. + Empty, + + /// The first database version. + /// Introduced: 2025-02-16. + Zero, + + /// Introduced: 2025-02-17. + One, + + /// Introduced: 2025-02-18. + Two, + + /// Introduced: 2025-03-21. + Three, +} +const CURRENT_VERSION: DbVersion = DbVersion::Three; + +async fn add_error_context( + function: impl Future<Output = Result<()>>, + level: DbVersion, +) -> Result<()> { + function + .await + .with_context(|| format!("Failed to migrate database to version: {level}")) +} + +async fn set_db_version( + tx: &mut Transaction<'_, Sqlite>, + old_version: Option<DbVersion>, + new_version: DbVersion, +) -> Result<()> { + let valid_from = get_current_date(); + + if let Some(old_version) = old_version { + let valid_to = valid_from + 1; + let old_version = old_version.as_sql_integer(); + + query!( + "UPDATE version SET valid_to = ? WHERE namespace = 'yt' AND number = ?;", + valid_to, + old_version + ) + .execute(&mut *(*tx)) + .await?; + } + + let version = new_version.as_sql_integer(); + + query!( + "INSERT INTO version (namespace, number, valid_from, valid_to) VALUES ('yt', ?, ?, NULL);", + version, + valid_from + ) + .execute(&mut *(*tx)) + .await?; + + Ok(()) +} + +impl DbVersion { + fn as_sql_integer(self) -> i32 { + match self { + DbVersion::Zero => 0, + DbVersion::One => 1, + DbVersion::Two => 2, + DbVersion::Three => 3, + + DbVersion::Empty => unreachable!("A empty version does not have an associated integer"), + } + } + + fn from_db(number: i64, namespace: &str) -> Result<Self> { + match (number, namespace) { + (0, "yt") => Ok(DbVersion::Zero), + (1, "yt") => Ok(DbVersion::One), + (2, "yt") => Ok(DbVersion::Two), + (3, "yt") => Ok(DbVersion::Three), + + (0, other) => bail!("Db version is Zero, but got unknown namespace: '{other}'"), + (1, other) => bail!("Db version is One, but got unknown namespace: '{other}'"), + (2, other) => bail!("Db version is Two, but got unknown namespace: '{other}'"), + (3, other) => bail!("Db version is Three, but got unknown namespace: '{other}'"), + + (other, "yt") => bail!("Got unkown version for 'yt' namespace: {other}"), + (num, nasp) => bail!("Got unkown version number ({num}) and namespace ('{nasp}')"), + } + } + + /// Try to update the database from version [`self`] to the [`CURRENT_VERSION`]. + /// + /// Each update is atomic, so if this function fails you are still guaranteed to have a + /// database at version `get_version`. + #[allow(clippy::too_many_lines)] + async fn update(self, app: &App) -> Result<()> { + match self { + Self::Empty => { + make_upgrade! {app, Self::Empty, Self::Zero, "./sql/0_Empty_to_Zero.sql"} + } + + Self::Zero => { + make_upgrade! {app, Self::Zero, Self::One, "./sql/1_Zero_to_One.sql"} + } + + Self::One => { + make_upgrade! {app, Self::One, Self::Two, "./sql/2_One_to_Two.sql"} + } + + Self::Two => { + make_upgrade! {app, Self::Two, Self::Three, "./sql/3_Two_to_Three.sql"} + } + + // This is the current_version + Self::Three => { + assert_eq!(self, CURRENT_VERSION); + assert_eq!(self, get_version(app).await?); + Ok(()) + } + } + } +} +impl Display for DbVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // It is a unit only enum, thus we can simply use the Debug formatting + <Self as std::fmt::Debug>::fmt(self, f) + } +} + +/// Returns the current data as UNIX time stamp. +fn get_current_date() -> i64 { + let start = SystemTime::now(); + let seconds_since_epoch: TimeDelta = TimeDelta::from_std( + start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"), + ) + .expect("Time does not go backwards"); + + // All database dates should be after the UNIX_EPOCH (and thus positiv) + seconds_since_epoch.num_milliseconds() +} + +/// Return the current database version. +/// +/// # Panics +/// Only if internal assertions fail. +pub async fn get_version(app: &App) -> Result<DbVersion> { + get_version_db(&app.database).await +} +/// Return the current database version. +/// +/// In contrast to the [`get_version`] function, this function does not +/// a fully instantiated [`App`], a database connection suffices. +/// +/// # Panics +/// Only if internal assertions fail. +pub async fn get_version_db(pool: &SqlitePool) -> Result<DbVersion> { + let version_table_exists = { + let query = query!( + "SELECT 1 as result FROM sqlite_master WHERE type = 'table' AND name = 'version'" + ) + .fetch_optional(pool) + .await?; + if let Some(output) = query { + assert_eq!(output.result, 1); + true + } else { + false + } + }; + if !version_table_exists { + return Ok(DbVersion::Empty); + } + + let current_version = query!( + " + SELECT namespace, number FROM version WHERE valid_to IS NULL; + " + ) + .fetch_one(pool) + .await + .context("Failed to fetch version number")?; + + DbVersion::from_db(current_version.number, current_version.namespace.as_str()) +} + +pub async fn migrate_db(app: &App) -> Result<()> { + let current_version = get_version(app) + .await + .context("Failed to determine initial version")?; + + if current_version == CURRENT_VERSION { + return Ok(()); + } + + info!("Migrate database from version '{current_version}' to version '{CURRENT_VERSION}'"); + + current_version.update(app).await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql new file mode 100644 index 0000000..d703bfc --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql @@ -0,0 +1,72 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +-- All tables should be declared STRICT, as I actually like to have types checking (and a +-- db that doesn't lie to me). + +-- Keep this table in sync with the `DbVersion` enumeration. +CREATE TABLE version ( + -- The `namespace` is only useful, if other tools ever build on this database + namespace TEXT NOT NULL, + + -- The version. + number INTEGER UNIQUE NOT NULL PRIMARY KEY, + + -- The validity of this version as UNIX time stamp + valid_from INTEGER NOT NULL CHECK (valid_from < valid_to), + -- If set to `NULL`, represents the current version + valid_to INTEGER UNIQUE CHECK (valid_to > valid_from) +) STRICT; + +-- Keep this table in sync with the `Video` structure +CREATE TABLE videos ( + cache_path TEXT UNIQUE CHECK (CASE WHEN cache_path IS NOT NULL THEN + status == 2 + ELSE + 1 + END), + description TEXT, + duration REAL, + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + last_status_change INTEGER NOT NULL, + parent_subscription_name TEXT, + priority INTEGER NOT NULL DEFAULT 0, + publish_date INTEGER, + status INTEGER NOT NULL DEFAULT 0 CHECK (status IN (0, 1, 2, 3, 4, 5) AND + CASE WHEN status == 2 THEN + cache_path IS NOT NULL + ELSE + 1 + END AND + CASE WHEN status != 2 THEN + cache_path IS NULL + ELSE + 1 + END), + status_change INTEGER NOT NULL DEFAULT 0 CHECK (status_change IN (0, 1)), + thumbnail_url TEXT, + title TEXT NOT NULL, + url TEXT UNIQUE NOT NULL +) STRICT; + +-- Store additional metadata for the videos marked to be watched +CREATE TABLE video_options ( + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + subtitle_langs TEXT NOT NULL, + playback_speed REAL NOT NULL, + FOREIGN KEY(extractor_hash) REFERENCES videos (extractor_hash) +) STRICT; + +-- Store subscriptions +CREATE TABLE subscriptions ( + name TEXT UNIQUE NOT NULL PRIMARY KEY, + url TEXT NOT NULL +) STRICT; diff --git a/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql new file mode 100644 index 0000000..da9315b --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql @@ -0,0 +1,28 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +-- Is the video currently in a playlist? +ALTER TABLE videos ADD in_playlist INTEGER NOT NULL DEFAULT 0 CHECK (in_playlist IN (0, 1)); +UPDATE videos SET in_playlist = 0; + +-- Is it 'focused' (i.e., the select video)? +-- Only of video should be focused at a time. +ALTER TABLE videos +ADD COLUMN is_focused INTEGER NOT NULL DEFAULT 0 +CHECK (is_focused IN (0, 1)); +UPDATE videos SET is_focused = 0; + +-- The progress the user made in watching the video. +ALTER TABLE videos ADD watch_progress INTEGER NOT NULL DEFAULT 0 CHECK (watch_progress <= duration); +-- Assume, that the user has watched the video to end, if it is marked as watched +UPDATE videos SET watch_progress = ifnull(duration, 0) WHERE status = 3; +UPDATE videos SET watch_progress = 0 WHERE status != 3; + +ALTER TABLE videos DROP COLUMN status_change; diff --git a/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql new file mode 100644 index 0000000..806de07 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql @@ -0,0 +1,11 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +ALTER TABLE videos DROP in_playlist; diff --git a/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql new file mode 100644 index 0000000..b33f849 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql @@ -0,0 +1,85 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + + +-- 1. Create new table +-- 2. Copy data +-- 3. Drop old table +-- 4. Rename new into old + +-- remove the original TRANSACTION +COMMIT TRANSACTION; + +-- tweak config +PRAGMA foreign_keys=OFF; + +-- start your own TRANSACTION +BEGIN TRANSACTION; + +CREATE TABLE videos_new ( + cache_path TEXT UNIQUE CHECK (CASE + WHEN cache_path IS NOT NULL THEN status == 2 + ELSE 1 + END), + description TEXT, + duration REAL, + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + last_status_change INTEGER NOT NULL, + parent_subscription_name TEXT, + priority INTEGER NOT NULL DEFAULT 0, + publish_date INTEGER, + status INTEGER NOT NULL DEFAULT 0 CHECK (status IN (0, 1, 2, 3, 4, 5) AND + CASE + WHEN status == 2 THEN cache_path IS NOT NULL + WHEN status != 2 THEN cache_path IS NULL + ELSE 1 + END), + thumbnail_url TEXT, + title TEXT NOT NULL, + url TEXT UNIQUE NOT NULL, + is_focused INTEGER UNIQUE DEFAULT NULL CHECK (CASE + WHEN is_focused IS NOT NULL THEN is_focused == 1 + ELSE 1 + END), + watch_progress INTEGER NOT NULL DEFAULT 0 CHECK (watch_progress <= duration) +) STRICT; + +INSERT INTO videos_new SELECT + videos.cache_path, + videos.description, + videos.duration, + videos.extractor_hash, + videos.last_status_change, + videos.parent_subscription_name, + videos.priority, + videos.publish_date, + videos.status, + videos.thumbnail_url, + videos.title, + videos.url, + dummy.is_focused, + videos.watch_progress +FROM videos, (SELECT NULL AS is_focused) AS dummy; + +DROP TABLE videos; + +ALTER TABLE videos_new RENAME TO videos; + +-- check foreign key constraint still upholding. +PRAGMA foreign_key_check; + +-- commit your own TRANSACTION +COMMIT TRANSACTION; + +-- rollback all config you setup before. +PRAGMA foreign_keys=ON; + +-- start a new TRANSACTION to let migrator commit it. +BEGIN TRANSACTION; diff --git a/crates/yt/src/storage/mod.rs b/crates/yt/src/storage/mod.rs new file mode 100644 index 0000000..d352b41 --- /dev/null +++ b/crates/yt/src/storage/mod.rs @@ -0,0 +1,14 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +pub mod migrate; +pub mod subscriptions; +pub mod video_database; diff --git a/crates/yt/src/storage/subscriptions.rs b/crates/yt/src/storage/subscriptions.rs new file mode 100644 index 0000000..6c0d08a --- /dev/null +++ b/crates/yt/src/storage/subscriptions.rs @@ -0,0 +1,141 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! Handle subscriptions + +use std::collections::HashMap; + +use anyhow::Result; +use log::debug; +use sqlx::query; +use url::Url; +use yt_dlp::YoutubeDLOptions; + +use crate::{app::App, unreachable::Unreachable}; + +#[derive(Clone, Debug)] +pub struct Subscription { + /// The human readable name of this subscription + pub name: String, + + /// The URL this subscription subscribes to + pub url: Url, +} + +impl Subscription { + #[must_use] + pub fn new(name: String, url: Url) -> Self { + Self { name, url } + } +} + +/// Check whether an URL could be used as a subscription URL +pub async fn check_url(url: Url) -> Result<bool> { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + debug!("{:#?}", info); + + Ok(info.get("_type") == Some(&serde_json::Value::String("Playlist".to_owned()))) +} + +#[derive(Default, Debug)] +pub struct Subscriptions(pub(crate) HashMap<String, Subscription>); + +/// Remove all subscriptions +pub async fn remove_all(app: &App) -> Result<()> { + query!( + " + DELETE FROM subscriptions; + ", + ) + .execute(&app.database) + .await?; + + Ok(()) +} + +/// Get a list of subscriptions +pub async fn get(app: &App) -> Result<Subscriptions> { + let raw_subs = query!( + " + SELECT * + FROM subscriptions; + " + ) + .fetch_all(&app.database) + .await?; + + let subscriptions: HashMap<String, Subscription> = raw_subs + .into_iter() + .map(|sub| { + ( + sub.name.clone(), + Subscription::new( + sub.name, + Url::parse(&sub.url).unreachable("It was an URL, when we inserted it."), + ), + ) + }) + .collect(); + + Ok(Subscriptions(subscriptions)) +} + +pub async fn add_subscription(app: &App, sub: &Subscription) -> Result<()> { + let url = sub.url.to_string(); + + query!( + " + INSERT INTO subscriptions ( + name, + url + ) VALUES (?, ?); + ", + sub.name, + url + ) + .execute(&app.database) + .await?; + + println!("Subscribed to '{}' at '{}'", sub.name, sub.url); + Ok(()) +} + +/// # Panics +/// Only if assertions fail +pub async fn remove_subscription(app: &App, sub: &Subscription) -> Result<()> { + let output = query!( + " + DELETE FROM subscriptions + WHERE name = ? + ", + sub.name, + ) + .execute(&app.database) + .await?; + + assert_eq!( + output.rows_affected(), + 1, + "The remove subscriptino query did effect more (or less) than one row. This is a bug." + ); + + println!("Unsubscribed from '{}' at '{}'", sub.name, sub.url); + + Ok(()) +} diff --git a/crates/yt/src/storage/video_database/downloader.rs b/crates/yt/src/storage/video_database/downloader.rs new file mode 100644 index 0000000..a95081e --- /dev/null +++ b/crates/yt/src/storage/video_database/downloader.rs @@ -0,0 +1,130 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use log::debug; +use sqlx::query; + +use crate::{ + app::App, + storage::video_database::{VideoStatus, VideoStatusMarker}, + unreachable::Unreachable, + video_from_record, +}; + +use super::{ExtractorHash, Video}; + +/// Returns to next video which should be downloaded. This respects the priority assigned by select. +/// It does not return videos, which are already cached. +/// +/// # Panics +/// Only if assertions fail. +pub async fn get_next_uncached_video(app: &App) -> Result<Option<Video>> { + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + // NOTE: The ORDER BY statement should be the same as the one in [`get::videos`].<2024-08-22> + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND cache_path IS NULL + ORDER BY priority DESC, publish_date DESC + LIMIT 1; + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record! {base})) + } +} + +/// Update the cached path of a video. Will be set to NULL if the path is None +/// This will also set the status to `Cached` when path is Some, otherwise it set's the status to +/// `Watch`. +pub async fn set_video_cache_path( + app: &App, + video: &ExtractorHash, + path: Option<&Path>, +) -> Result<()> { + if let Some(path) = path { + debug!( + "Setting cache path from '{}' to '{}'", + video.into_short_hash(app).await?, + path.display() + ); + + let path_str = path.display().to_string(); + let extractor_hash = video.hash().to_string(); + let status = VideoStatusMarker::Cached.as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = ?, status = ? + WHERE extractor_hash = ?; + "#, + path_str, + status, + extractor_hash + ) + .execute(&app.database) + .await?; + + Ok(()) + } else { + debug!( + "Setting cache path from '{}' to NULL", + video.into_short_hash(app).await?, + ); + + let extractor_hash = video.hash().to_string(); + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = NULL, status = ? + WHERE extractor_hash = ?; + "#, + status, + extractor_hash + ) + .execute(&app.database) + .await?; + + Ok(()) + } +} + +/// Returns the number of cached videos +pub async fn get_allocated_cache(app: &App) -> Result<u32> { + let count = query!( + r#" + SELECT COUNT(cache_path) as count + FROM videos + WHERE cache_path IS NOT NULL; +"#, + ) + .fetch_one(&app.database) + .await?; + + Ok(u32::try_from(count.count) + .unreachable("The value should be strictly positive (and bolow `u32::Max`)")) +} diff --git a/crates/yt/src/storage/video_database/extractor_hash.rs b/crates/yt/src/storage/video_database/extractor_hash.rs new file mode 100644 index 0000000..df545d7 --- /dev/null +++ b/crates/yt/src/storage/video_database/extractor_hash.rs @@ -0,0 +1,163 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashSet, fmt::Display, str::FromStr}; + +use anyhow::{Context, Result, bail}; +use blake3::Hash; +use log::debug; +use tokio::sync::OnceCell; + +use crate::{app::App, storage::video_database::get::get_all_hashes, unreachable::Unreachable}; + +static EXTRACTOR_HASH_LENGTH: OnceCell<usize> = OnceCell::const_new(); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] +pub struct ExtractorHash { + hash: Hash, +} + +impl Display for ExtractorHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.hash.fmt(f) + } +} + +#[derive(Debug, Clone)] +pub struct ShortHash(String); + +impl Display for ShortHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone)] +#[allow(clippy::module_name_repetitions)] +pub struct LazyExtractorHash { + value: ShortHash, +} + +impl FromStr for LazyExtractorHash { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result<Self, Self::Err> { + // perform some cheap validation + if s.len() > 64 { + bail!("A hash can only contain 64 bytes!"); + } + + Ok(Self { + value: ShortHash(s.to_owned()), + }) + } +} + +impl LazyExtractorHash { + /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] + pub async fn realize(self, app: &App) -> Result<ExtractorHash> { + ExtractorHash::from_short_hash(app, &self.value).await + } +} + +impl ExtractorHash { + #[must_use] + pub fn from_hash(hash: Hash) -> Self { + Self { hash } + } + pub async fn from_short_hash(app: &App, s: &ShortHash) -> Result<Self> { + Ok(Self { + hash: Self::short_hash_to_full_hash(app, s).await?, + }) + } + + #[must_use] + pub fn hash(&self) -> &Hash { + &self.hash + } + + pub async fn into_short_hash(&self, app: &App) -> Result<ShortHash> { + let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { + *needed_chars + } else { + let needed_chars = self + .get_needed_char_len(app) + .await + .context("Failed to calculate needed char length")?; + EXTRACTOR_HASH_LENGTH.set(needed_chars).unreachable( + "This should work at this stage, as we checked above that it is empty.", + ); + + needed_chars + }; + + Ok(ShortHash( + self.hash() + .to_hex() + .chars() + .take(needed_chars) + .collect::<String>(), + )) + } + + async fn short_hash_to_full_hash(app: &App, s: &ShortHash) -> Result<Hash> { + let all_hashes = get_all_hashes(app) + .await + .context("Failed to fetch all extractor -hashesh from database")?; + + let needed_chars = s.0.len(); + + for hash in all_hashes { + if hash.to_hex()[..needed_chars] == s.0 { + return Ok(hash); + } + } + + bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); + } + + async fn get_needed_char_len(&self, app: &App) -> Result<usize> { + debug!("Calculating the needed hash char length"); + let all_hashes = get_all_hashes(app) + .await + .context("Failed to fetch all extractor -hashesh from database")?; + + let all_char_vec_hashes = all_hashes + .into_iter() + .map(|hash| hash.to_hex().chars().collect::<Vec<char>>()) + .collect::<Vec<Vec<_>>>(); + + // This value should be updated later, if not rust will panic in the assertion. + let mut needed_chars: usize = 1000; + 'outer: for i in 1..64 { + let i_chars: Vec<String> = all_char_vec_hashes + .iter() + .map(|vec| vec.iter().take(i).collect::<String>()) + .collect(); + + let mut uniqnes_hashmap: HashSet<String> = HashSet::new(); + for ch in i_chars { + if !uniqnes_hashmap.insert(ch) { + // The key was already in the hash map, thus we have a duplicated char and need + // at least one char more + continue 'outer; + } + } + + needed_chars = i; + break 'outer; + } + + assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); + + Ok(needed_chars) + } +} diff --git a/crates/yt/src/storage/video_database/get/mod.rs b/crates/yt/src/storage/video_database/get/mod.rs new file mode 100644 index 0000000..0456cd3 --- /dev/null +++ b/crates/yt/src/storage/video_database/get/mod.rs @@ -0,0 +1,307 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! These functions interact with the storage db in a read-only way. They are added on-demand (as +//! you could theoretically just could do everything with the `get_videos` function), as +//! performance or convince requires. +use std::{fs::File, path::PathBuf}; + +use anyhow::{Context, Result, bail}; +use blake3::Hash; +use log::{debug, trace}; +use sqlx::query; +use yt_dlp::InfoJson; + +use crate::{ + app::App, + storage::{ + subscriptions::Subscription, + video_database::{Video, extractor_hash::ExtractorHash}, + }, + unreachable::Unreachable, +}; + +use super::{MpvOptions, VideoOptions, VideoStatus, VideoStatusMarker, YtDlpOptions}; + +mod playlist; +pub use playlist::*; + +#[macro_export] +macro_rules! video_from_record { + ($record:expr) => { + Video { + description: $record.description.clone(), + duration: $crate::storage::video_database::MaybeDuration::from_maybe_secs_f64( + $record.duration, + ), + extractor_hash: + $crate::storage::video_database::extractor_hash::ExtractorHash::from_hash( + $record + .extractor_hash + .parse() + .expect("The db hash should be a valid blake3 hash"), + ), + last_status_change: $crate::storage::video_database::TimeStamp::from_secs( + $record.last_status_change, + ), + parent_subscription_name: $record.parent_subscription_name.clone(), + publish_date: $record + .publish_date + .map(|pd| $crate::storage::video_database::TimeStamp::from_secs(pd)), + status: { + let marker = $crate::storage::video_database::VideoStatusMarker::from_db_integer( + $record.status, + ); + + let optional = if let Some(cache_path) = &$record.cache_path { + Some(( + PathBuf::from(cache_path), + if $record.is_focused == Some(1) { + true + } else { + false + }, + )) + } else { + None + }; + + $crate::storage::video_database::VideoStatus::from_marker(marker, optional) + }, + thumbnail_url: if let Some(url) = &$record.thumbnail_url { + Some(url::Url::parse(&url).expect("Parsing this as url should always work")) + } else { + None + }, + title: $record.title.clone(), + url: url::Url::parse(&$record.url).expect("Parsing this as url should always work"), + priority: $crate::storage::video_database::Priority::from($record.priority), + + watch_progress: std::time::Duration::from_secs( + u64::try_from($record.watch_progress).expect("The record is positive i64"), + ), + } + }; +} + +/// Returns the videos that are in the `allowed_states`. +/// +/// # Panics +/// Only, if assertions fail. +pub async fn videos(app: &App, allowed_states: &[VideoStatusMarker]) -> Result<Vec<Video>> { + fn test(all_states: &[VideoStatusMarker], check: VideoStatusMarker) -> Option<i64> { + if all_states.contains(&check) { + trace!("State '{check:?}' marked as active"); + Some(check.as_db_integer()) + } else { + trace!("State '{check:?}' marked as inactive"); + None + } + } + fn states_to_string(allowed_states: &[VideoStatusMarker]) -> String { + let mut states = allowed_states + .iter() + .fold(String::from("&["), |mut acc, state| { + acc.push_str(state.as_str()); + acc.push_str(", "); + acc + }); + states = states.trim().to_owned(); + states = states.trim_end_matches(',').to_owned(); + states.push(']'); + states + } + + debug!( + "Fetching videos in the states: '{}'", + states_to_string(allowed_states) + ); + let active_pick: Option<i64> = test(allowed_states, VideoStatusMarker::Pick); + let active_watch: Option<i64> = test(allowed_states, VideoStatusMarker::Watch); + let active_cached: Option<i64> = test(allowed_states, VideoStatusMarker::Cached); + let active_watched: Option<i64> = test(allowed_states, VideoStatusMarker::Watched); + let active_drop: Option<i64> = test(allowed_states, VideoStatusMarker::Drop); + let active_dropped: Option<i64> = test(allowed_states, VideoStatusMarker::Dropped); + + let videos = query!( + r" + SELECT * + FROM videos + WHERE status IN (?,?,?,?,?,?) + ORDER BY priority DESC, publish_date DESC; + ", + active_pick, + active_watch, + active_cached, + active_watched, + active_drop, + active_dropped, + ) + .fetch_all(&app.database) + .await + .with_context(|| { + format!( + "Failed to query videos with states: '{}'", + states_to_string(allowed_states) + ) + })?; + + let real_videos: Vec<Video> = videos + .iter() + .map(|base| -> Video { + video_from_record! {base} + }) + .collect(); + + Ok(real_videos) +} + +pub fn video_info_json(video: &Video) -> Result<Option<InfoJson>> { + if let VideoStatus::Cached { mut cache_path, .. } = video.status.clone() { + if !cache_path.set_extension("info.json") { + bail!( + "Failed to change path extension to 'info.json': {}", + cache_path.display() + ); + } + let info_json_string = File::open(cache_path)?; + let info_json: InfoJson = serde_json::from_reader(&info_json_string)?; + + Ok(Some(info_json)) + } else { + Ok(None) + } +} + +pub async fn video_by_hash(app: &App, hash: &ExtractorHash) -> Result<Video> { + let ehash = hash.hash().to_string(); + + let raw_video = query!( + " + SELECT * FROM videos WHERE extractor_hash = ?; + ", + ehash + ) + .fetch_one(&app.database) + .await?; + + Ok(video_from_record! {raw_video}) +} + +pub async fn get_all_hashes(app: &App) -> Result<Vec<Hash>> { + let hashes_hex = query!( + r#" + SELECT extractor_hash + FROM videos; + "# + ) + .fetch_all(&app.database) + .await?; + + Ok(hashes_hex + .iter() + .map(|hash| { + Hash::from_hex(&hash.extractor_hash).unreachable( + "These values started as blake3 hashes, they should stay blake3 hashes", + ) + }) + .collect()) +} + +pub async fn get_video_hashes(app: &App, subs: &Subscription) -> Result<Vec<Hash>> { + let hashes_hex = query!( + r#" + SELECT extractor_hash + FROM videos + WHERE parent_subscription_name = ?; + "#, + subs.name + ) + .fetch_all(&app.database) + .await?; + + Ok(hashes_hex + .iter() + .map(|hash| { + Hash::from_hex(&hash.extractor_hash).unreachable( + "These values started as blake3 hashes, they should stay blake3 hashes", + ) + }) + .collect()) +} + +pub async fn get_video_yt_dlp_opts(app: &App, hash: &ExtractorHash) -> Result<YtDlpOptions> { + let ehash = hash.hash().to_string(); + + let yt_dlp_options = query!( + r#" + SELECT subtitle_langs + FROM video_options + WHERE extractor_hash = ?; + "#, + ehash + ) + .fetch_one(&app.database) + .await + .with_context(|| { + format!("Failed to fetch the `yt_dlp_video_opts` for video with hash: '{hash}'",) + })?; + + Ok(YtDlpOptions { + subtitle_langs: yt_dlp_options.subtitle_langs, + }) +} +pub async fn video_mpv_opts(app: &App, hash: &ExtractorHash) -> Result<MpvOptions> { + let ehash = hash.hash().to_string(); + + let mpv_options = query!( + r#" + SELECT playback_speed + FROM video_options + WHERE extractor_hash = ?; + "#, + ehash + ) + .fetch_one(&app.database) + .await + .with_context(|| { + format!("Failed to fetch the `mpv_video_opts` for video with hash: '{hash}'") + })?; + + Ok(MpvOptions { + playback_speed: mpv_options.playback_speed, + }) +} + +pub async fn get_video_opts(app: &App, hash: &ExtractorHash) -> Result<VideoOptions> { + let ehash = hash.hash().to_string(); + + let opts = query!( + r#" + SELECT playback_speed, subtitle_langs + FROM video_options + WHERE extractor_hash = ?; + "#, + ehash + ) + .fetch_one(&app.database) + .await + .with_context(|| format!("Failed to fetch the `video_opts` for video with hash: '{hash}'"))?; + + let mpv = MpvOptions { + playback_speed: opts.playback_speed, + }; + let yt_dlp = YtDlpOptions { + subtitle_langs: opts.subtitle_langs, + }; + + Ok(VideoOptions { yt_dlp, mpv }) +} diff --git a/crates/yt/src/storage/video_database/get/playlist/iterator.rs b/crates/yt/src/storage/video_database/get/playlist/iterator.rs new file mode 100644 index 0000000..4c45bf7 --- /dev/null +++ b/crates/yt/src/storage/video_database/get/playlist/iterator.rs @@ -0,0 +1,101 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, +}; + +use crate::storage::video_database::{Video, VideoStatus}; + +use super::Playlist; + +/// Turn a cached video into it's `cache_path` +fn to_cache_video(video: Video) -> PathBuf { + if let VideoStatus::Cached { cache_path, .. } = video.status { + cache_path + } else { + unreachable!("ALl of these videos should be cached.") + } +} + +#[derive(Debug)] +pub struct PlaylistIterator { + paths: VecDeque<PathBuf>, +} + +impl Iterator for PlaylistIterator { + type Item = <Playlist as IntoIterator>::Item; + + fn next(&mut self) -> Option<Self::Item> { + self.paths.pop_front() + } +} + +impl DoubleEndedIterator for PlaylistIterator { + fn next_back(&mut self) -> Option<Self::Item> { + self.paths.pop_back() + } +} + +impl IntoIterator for Playlist { + type Item = PathBuf; + + type IntoIter = PlaylistIterator; + + fn into_iter(self) -> Self::IntoIter { + let paths = self.videos.into_iter().map(to_cache_video).collect(); + Self::IntoIter { paths } + } +} + +#[derive(Debug)] +pub struct PlaylistIteratorBorrowed<'a> { + paths: Vec<&'a Path>, + index: usize, +} + +impl<'a> Iterator for PlaylistIteratorBorrowed<'a> { + type Item = <&'a Playlist as IntoIterator>::Item; + + fn next(&mut self) -> Option<Self::Item> { + let output = self.paths.get(self.index); + self.index += 1; + output.map(|v| &**v) + } +} + +impl<'a> Playlist { + #[must_use] + pub fn iter(&'a self) -> PlaylistIteratorBorrowed<'a> { + <&Self as IntoIterator>::into_iter(self) + } +} + +impl<'a> IntoIterator for &'a Playlist { + type Item = &'a Path; + + type IntoIter = PlaylistIteratorBorrowed<'a>; + + fn into_iter(self) -> Self::IntoIter { + let paths = self + .videos + .iter() + .map(|vid| { + if let VideoStatus::Cached { cache_path, .. } = &vid.status { + cache_path.as_path() + } else { + unreachable!("ALl of these videos should be cached.") + } + }) + .collect(); + Self::IntoIter { paths, index: 0 } + } +} diff --git a/crates/yt/src/storage/video_database/get/playlist/mod.rs b/crates/yt/src/storage/video_database/get/playlist/mod.rs new file mode 100644 index 0000000..f6aadbf --- /dev/null +++ b/crates/yt/src/storage/video_database/get/playlist/mod.rs @@ -0,0 +1,167 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! This file contains the getters for the internal playlist + +use std::{ops::Add, path::PathBuf}; + +use crate::{ + app::App, + storage::video_database::{Video, VideoStatusMarker, extractor_hash::ExtractorHash}, + video_from_record, +}; + +use anyhow::Result; +use sqlx::query; + +pub mod iterator; + +/// Zero-based index into the internal playlist. +#[derive(Debug, Clone, Copy)] +pub struct PlaylistIndex(usize); + +impl From<PlaylistIndex> for usize { + fn from(value: PlaylistIndex) -> Self { + value.0 + } +} + +impl From<usize> for PlaylistIndex { + fn from(value: usize) -> Self { + Self(value) + } +} + +impl Add<usize> for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: usize) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl Add for PlaylistIndex { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +/// A representation of the internal Playlist +#[derive(Debug)] +pub struct Playlist { + videos: Vec<Video>, +} + +impl Playlist { + /// Return the videos of this playlist. + #[must_use] + pub fn as_videos(&self) -> &[Video] { + &self.videos + } + + /// Turn this playlist to it's videos + #[must_use] + pub fn to_videos(self) -> Vec<Video> { + self.videos + } + + /// Find the index of the video specified by the `video_hash`. + /// + /// # Panics + /// Only if internal assertions fail. + #[must_use] + pub fn find_index(&self, video_hash: &ExtractorHash) -> Option<PlaylistIndex> { + if let Some((index, value)) = self + .videos + .iter() + .enumerate() + .find(|(_, other)| other.extractor_hash == *video_hash) + { + assert_eq!(value.extractor_hash, *video_hash); + Some(PlaylistIndex(index)) + } else { + None + } + } + + /// Select a video based on it's index + #[must_use] + pub fn get(&self, index: PlaylistIndex) -> Option<&Video> { + self.videos.get(index.0) + } + + /// Returns the number of videos in the playlist + #[must_use] + pub fn len(&self) -> usize { + self.videos.len() + } + /// Is the playlist empty? + #[must_use] + pub fn is_empty(&self) -> bool { + self.videos.is_empty() + } +} + +/// Return the current playlist index. +/// +/// This effectively looks for the currently focused video and returns it's index. +/// +/// # Panics +/// Only if internal assertions fail. +pub async fn current_playlist_index(app: &App) -> Result<Option<PlaylistIndex>> { + if let Some(focused) = currently_focused_video(app).await? { + let playlist = playlist(app).await?; + let index = playlist + .find_index(&focused.extractor_hash) + .expect("All focused videos must also be in the playlist"); + Ok(Some(index)) + } else { + Ok(None) + } +} + +/// Return the video in the playlist at the position `index`. +pub async fn playlist_entry(app: &App, index: PlaylistIndex) -> Result<Option<Video>> { + let playlist = playlist(app).await?; + + if let Some(vid) = playlist.get(index) { + Ok(Some(vid.to_owned())) + } else { + Ok(None) + } +} + +pub async fn playlist(app: &App) -> Result<Playlist> { + let videos = super::videos(app, &[VideoStatusMarker::Cached]).await?; + + Ok(Playlist { videos }) +} + +/// This returns the video with the `is_focused` flag set. +/// # Panics +/// Only if assertions fail. +pub async fn currently_focused_video(app: &App) -> Result<Option<Video>> { + let cached_status = VideoStatusMarker::Cached.as_db_integer(); + let record = query!( + "SELECT * FROM videos WHERE is_focused = 1 AND status = ?", + cached_status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = record { + Ok(None) + } else { + let base = record?; + Ok(Some(video_from_record! {base})) + } +} diff --git a/crates/yt/src/storage/video_database/mod.rs b/crates/yt/src/storage/video_database/mod.rs new file mode 100644 index 0000000..74d09f0 --- /dev/null +++ b/crates/yt/src/storage/video_database/mod.rs @@ -0,0 +1,329 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + fmt::{Display, Write}, + path::PathBuf, + time::Duration, +}; + +use chrono::{DateTime, Utc}; +use url::Url; + +use crate::{ + app::App, select::selection_file::duration::MaybeDuration, + storage::video_database::extractor_hash::ExtractorHash, +}; + +pub mod downloader; +pub mod extractor_hash; +pub mod get; +pub mod notify; +pub mod set; + +#[derive(Debug, Clone)] +pub struct Video { + pub description: Option<String>, + pub duration: MaybeDuration, + pub extractor_hash: ExtractorHash, + pub last_status_change: TimeStamp, + + /// The associated subscription this video was fetched from (null, when the video was `add`ed) + pub parent_subscription_name: Option<String>, + pub priority: Priority, + pub publish_date: Option<TimeStamp>, + pub status: VideoStatus, + pub thumbnail_url: Option<Url>, + pub title: String, + pub url: Url, + + /// The seconds the user has already watched the video + pub watch_progress: Duration, +} + +/// The priority of a [`Video`]. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Priority { + value: i64, +} +impl Priority { + /// Return the underlying value to insert that into the database + #[must_use] + pub fn as_db_integer(&self) -> i64 { + self.value + } +} +impl From<i64> for Priority { + fn from(value: i64) -> Self { + Self { value } + } +} +impl Display for Priority { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.value.fmt(f) + } +} + +/// An UNIX time stamp. +#[derive(Debug, Default, Clone, Copy)] +pub struct TimeStamp { + value: i64, +} +impl TimeStamp { + /// Return the seconds since the UNIX epoch for this [`TimeStamp`]. + #[must_use] + pub fn as_secs(&self) -> i64 { + self.value + } + + /// Construct a [`TimeStamp`] from a count of seconds since the UNIX epoch. + #[must_use] + pub fn from_secs(value: i64) -> Self { + Self { value } + } + + /// Construct a [`TimeStamp`] from the current time. + #[must_use] + pub fn from_now() -> Self { + Self { + value: Utc::now().timestamp(), + } + } +} +impl Display for TimeStamp { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + DateTime::from_timestamp(self.value, 0) + .expect("The timestamps should always be valid") + .format("%Y-%m-%d") + .fmt(f) + } +} + +#[derive(Debug)] +pub struct VideoOptions { + pub yt_dlp: YtDlpOptions, + pub mpv: MpvOptions, +} +impl VideoOptions { + pub(crate) fn new(subtitle_langs: String, playback_speed: f64) -> Self { + let yt_dlp = YtDlpOptions { subtitle_langs }; + let mpv = MpvOptions { playback_speed }; + Self { yt_dlp, mpv } + } + + /// This will write out the options that are different from the defaults. + /// Beware, that this does not set the priority. + #[must_use] + pub fn to_cli_flags(self, app: &App) -> String { + let mut f = String::new(); + + if (self.mpv.playback_speed - app.config.select.playback_speed).abs() > f64::EPSILON { + write!(f, " --speed '{}'", self.mpv.playback_speed).expect("Works"); + } + if self.yt_dlp.subtitle_langs != app.config.select.subtitle_langs { + write!(f, " --subtitle-langs '{}'", self.yt_dlp.subtitle_langs).expect("Works"); + } + + f.trim().to_owned() + } +} + +#[derive(Debug, Clone, Copy)] +/// Additionally settings passed to mpv on watch +pub struct MpvOptions { + /// The playback speed. (1 is 100%, 2.7 is 270%, and so on) + pub playback_speed: f64, +} + +#[derive(Debug)] +/// Additionally configuration options, passed to yt-dlp on download +pub struct YtDlpOptions { + /// In the form of `lang1,lang2,lang3` (e.g. `en,de,sv`) + pub subtitle_langs: String, +} + +/// # Video Lifetime (words in <brackets> are commands): +/// <Pick> +/// / \ +/// <Watch> <Drop> -> Dropped // yt select +/// | +/// Cache // yt cache +/// | +/// Watched // yt watch +#[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub enum VideoStatus { + #[default] + Pick, + + /// The video has been select to be watched + Watch, + /// The video has been cached and is ready to be watched + Cached { + cache_path: PathBuf, + is_focused: bool, + }, + /// The video has been watched + Watched, + + /// The video has been select to be dropped + Drop, + /// The video has been dropped + Dropped, +} + +impl VideoStatus { + /// Reconstruct a [`VideoStatus`] for it's marker and the optional parts. + /// This should only be used by the db record to [`Video`] code. + /// + /// # Panics + /// Only if internal expectations fail. + #[must_use] + pub fn from_marker(marker: VideoStatusMarker, optional: Option<(PathBuf, bool)>) -> Self { + match marker { + VideoStatusMarker::Pick => Self::Pick, + VideoStatusMarker::Watch => Self::Watch, + VideoStatusMarker::Cached => { + let (cache_path, is_focused) = + optional.expect("This should be some, when the video status is cached"); + Self::Cached { + cache_path, + is_focused, + } + } + VideoStatusMarker::Watched => Self::Watched, + VideoStatusMarker::Drop => Self::Drop, + VideoStatusMarker::Dropped => Self::Dropped, + } + } + + /// Turn the [`VideoStatus`] to its internal parts. This is only really useful for the database + /// functions. + #[must_use] + pub fn to_parts_for_db(self) -> (VideoStatusMarker, Option<(PathBuf, bool)>) { + match self { + VideoStatus::Pick => (VideoStatusMarker::Pick, None), + VideoStatus::Watch => (VideoStatusMarker::Watch, None), + VideoStatus::Cached { + cache_path, + is_focused, + } => (VideoStatusMarker::Cached, Some((cache_path, is_focused))), + VideoStatus::Watched => (VideoStatusMarker::Watched, None), + VideoStatus::Drop => (VideoStatusMarker::Drop, None), + VideoStatus::Dropped => (VideoStatusMarker::Dropped, None), + } + } + + /// Return the associated [`VideoStatusMarker`] for this [`VideoStatus`]. + #[must_use] + pub fn as_marker(&self) -> VideoStatusMarker { + match self { + VideoStatus::Pick => VideoStatusMarker::Pick, + VideoStatus::Watch => VideoStatusMarker::Watch, + VideoStatus::Cached { .. } => VideoStatusMarker::Cached, + VideoStatus::Watched => VideoStatusMarker::Watched, + VideoStatus::Drop => VideoStatusMarker::Drop, + VideoStatus::Dropped => VideoStatusMarker::Dropped, + } + } +} + +/// Unit only variant of [`VideoStatus`] +#[derive(Default, Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] +pub enum VideoStatusMarker { + #[default] + Pick, + + /// The video has been select to be watched + Watch, + /// The video has been cached and is ready to be watched + Cached, + /// The video has been watched + Watched, + + /// The video has been select to be dropped + Drop, + /// The video has been dropped + Dropped, +} + +impl VideoStatusMarker { + pub const ALL: &'static [Self; 6] = &[ + Self::Pick, + // + Self::Watch, + Self::Cached, + Self::Watched, + // + Self::Drop, + Self::Dropped, + ]; + + #[must_use] + pub fn as_command(&self) -> &str { + // NOTE: Keep the serialize able variants synced with the main `select` function <2024-06-14> + // Also try to ensure, that the strings have the same length + match self { + Self::Pick => "pick ", + + Self::Watch | Self::Cached => "watch ", + Self::Watched => "watched", + + Self::Drop | Self::Dropped => "drop ", + } + } + + #[must_use] + pub fn as_db_integer(&self) -> i64 { + // These numbers should not change their mapping! + // Oh, and keep them in sync with the SQLite check constraint. + match self { + Self::Pick => 0, + + Self::Watch => 1, + Self::Cached => 2, + Self::Watched => 3, + + Self::Drop => 4, + Self::Dropped => 5, + } + } + #[must_use] + pub fn from_db_integer(num: i64) -> Self { + match num { + 0 => Self::Pick, + + 1 => Self::Watch, + 2 => Self::Cached, + 3 => Self::Watched, + + 4 => Self::Drop, + 5 => Self::Dropped, + other => unreachable!( + "The database returned a enum discriminator, unknown to us: '{}'", + other + ), + } + } + + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Pick => "Pick", + + Self::Watch => "Watch", + Self::Cached => "Cache", + Self::Watched => "Watched", + + Self::Drop => "Drop", + Self::Dropped => "Dropped", + } + } +} diff --git a/crates/yt/src/storage/video_database/notify.rs b/crates/yt/src/storage/video_database/notify.rs new file mode 100644 index 0000000..b55c00a --- /dev/null +++ b/crates/yt/src/storage/video_database/notify.rs @@ -0,0 +1,77 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + path::{Path, PathBuf}, + sync::mpsc, + thread::sleep, + time::Duration, +}; + +use crate::app::App; + +use anyhow::{Context, Result}; +use notify::{ + Event, EventKind, RecursiveMode, Watcher, + event::{DataChange, ModifyKind}, +}; +use tokio::task; + +/// This functions registers a watcher for the database and only returns once a write was +/// registered for the database. +pub async fn wait_for_db_write(app: &App) -> Result<()> { + let db_path: PathBuf = app.config.paths.database_path.clone(); + task::spawn_blocking(move || wait_for_db_write_sync(&db_path)).await? +} + +fn wait_for_db_write_sync(db_path: &Path) -> Result<()> { + let (tx, rx) = mpsc::channel::<notify::Result<Event>>(); + + let mut watcher = notify::recommended_watcher(tx)?; + + watcher.watch(db_path, RecursiveMode::NonRecursive)?; + + for res in rx { + let event = res.context("Failed to wait for db write")?; + + if let EventKind::Modify(ModifyKind::Data(DataChange::Any)) = event.kind { + // Buffer some of the `Modify` event burst. + sleep(Duration::from_millis(10)); + + return Ok(()); + } + } + + Ok(()) +} + +/// This functions registers a watcher for the cache path and returns once a file was removed +pub async fn wait_for_cache_reduction(app: &App) -> Result<()> { + let download_directory: PathBuf = app.config.paths.download_dir.clone(); + task::spawn_blocking(move || wait_for_cache_reduction_sync(&download_directory)).await? +} + +fn wait_for_cache_reduction_sync(download_directory: &Path) -> Result<()> { + let (tx, rx) = mpsc::channel::<notify::Result<Event>>(); + + let mut watcher = notify::recommended_watcher(tx)?; + + watcher.watch(download_directory, RecursiveMode::Recursive)?; + + for res in rx { + let event = res.context("Failed to wait for cache size reduction")?; + + if let EventKind::Remove(_) = event.kind { + return Ok(()); + } + } + + Ok(()) +} diff --git a/crates/yt/src/storage/video_database/set/mod.rs b/crates/yt/src/storage/video_database/set/mod.rs new file mode 100644 index 0000000..8c1be4a --- /dev/null +++ b/crates/yt/src/storage/video_database/set/mod.rs @@ -0,0 +1,333 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +//! These functions change the database. They are added on a demand basis. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use chrono::Utc; +use log::{debug, info}; +use sqlx::query; +use tokio::fs; + +use crate::{app::App, storage::video_database::extractor_hash::ExtractorHash, video_from_record}; + +use super::{Priority, Video, VideoOptions, VideoStatus}; + +mod playlist; +pub use playlist::*; + +const fn is_focused_to_value(is_focused: bool) -> Option<i8> { + if is_focused { Some(1) } else { None } +} + +/// Set a new status for a video. +/// This will only update the status time stamp/priority when the status or the priority has changed . +pub async fn video_status( + app: &App, + video_hash: &ExtractorHash, + new_status: VideoStatus, + new_priority: Option<Priority>, +) -> Result<()> { + let video_hash = video_hash.hash().to_string(); + + let old = { + let base = query!( + r#" + SELECT * + FROM videos + WHERE extractor_hash = ? + "#, + video_hash + ) + .fetch_one(&app.database) + .await?; + + video_from_record! {base} + }; + + let old_marker = old.status.as_marker(); + let (cache_path, is_focused) = { + fn cache_path_to_string(path: &Path) -> Result<String> { + Ok(path + .to_str() + .with_context(|| { + format!( + "Failed to parse cache path ('{}') as utf8 string", + path.display() + ) + })? + .to_owned()) + } + + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &new_status + { + ( + Some(cache_path_to_string(cache_path)?), + is_focused_to_value(*is_focused), + ) + } else { + (None, None) + } + }; + + let new_status = new_status.as_marker(); + + if let Some(new_priority) = new_priority { + if old_marker == new_status && old.priority == new_priority { + return Ok(()); + } + + let now = Utc::now().timestamp(); + + debug!( + "Running status change: {:#?} -> {:#?}...", + old_marker, new_status, + ); + + let new_status = new_status.as_db_integer(); + let new_priority = new_priority.as_db_integer(); + query!( + r#" + UPDATE videos + SET status = ?, last_status_change = ?, priority = ?, cache_path = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + new_status, + now, + new_priority, + cache_path, + is_focused, + video_hash + ) + .execute(&app.database) + .await?; + } else { + if old_marker == new_status { + return Ok(()); + } + + let now = Utc::now().timestamp(); + + debug!( + "Running status change: {:#?} -> {:#?}...", + old_marker, new_status, + ); + + let new_status = new_status.as_db_integer(); + query!( + r#" + UPDATE videos + SET status = ?, last_status_change = ?, cache_path = ?, is_focused = ? + WHERE extractor_hash = ?; + "#, + new_status, + now, + cache_path, + is_focused, + video_hash + ) + .execute(&app.database) + .await?; + } + + debug!("Finished status change."); + Ok(()) +} + +/// Mark a video as watched. +/// This will both set the status to `Watched` and the `cache_path` to Null. +/// +/// # Panics +/// Only if assertions fail. +pub async fn video_watched(app: &App, video: &ExtractorHash) -> Result<()> { + let old = { + let video_hash = video.hash().to_string(); + + let base = query!( + r#" + SELECT * + FROM videos + WHERE extractor_hash = ? + "#, + video_hash + ) + .fetch_one(&app.database) + .await?; + + video_from_record! {base} + }; + + info!("Will set video watched: '{}'", old.title); + + if let VideoStatus::Cached { cache_path, .. } = &old.status { + if let Ok(true) = cache_path.try_exists() { + fs::remove_file(cache_path).await?; + } + } else { + unreachable!("The video must be marked as Cached before it can be marked Watched"); + } + + video_status(app, video, VideoStatus::Watched, None).await?; + + Ok(()) +} + +pub(crate) async fn video_watch_progress( + app: &App, + extractor_hash: &ExtractorHash, + watch_progress: u32, +) -> std::result::Result<(), anyhow::Error> { + let video_extractor_hash = extractor_hash.hash().to_string(); + + query!( + r#" + UPDATE videos + SET watch_progress = ? + WHERE extractor_hash = ?; + "#, + watch_progress, + video_extractor_hash, + ) + .execute(&app.database) + .await?; + + Ok(()) +} + +pub async fn set_video_options( + app: &App, + hash: &ExtractorHash, + video_options: &VideoOptions, +) -> Result<()> { + let video_extractor_hash = hash.hash().to_string(); + let playback_speed = video_options.mpv.playback_speed; + let subtitle_langs = &video_options.yt_dlp.subtitle_langs; + + query!( + r#" + UPDATE video_options + SET playback_speed = ?, subtitle_langs = ? + WHERE extractor_hash = ?; + "#, + playback_speed, + subtitle_langs, + video_extractor_hash, + ) + .execute(&app.database) + .await?; + + Ok(()) +} + +/// # Panics +/// Only if internal expectations fail. +pub async fn add_video(app: &App, video: Video) -> Result<()> { + let parent_subscription_name = video.parent_subscription_name; + + let thumbnail_url = video.thumbnail_url.map(|val| val.to_string()); + + let url = video.url.to_string(); + let extractor_hash = video.extractor_hash.hash().to_string(); + + let default_subtitle_langs = &app.config.select.subtitle_langs; + let default_mpv_playback_speed = app.config.select.playback_speed; + + let status = video.status.as_marker().as_db_integer(); + let (cache_path, is_focused) = if let VideoStatus::Cached { + cache_path, + is_focused, + } = video.status + { + ( + Some( + cache_path + .to_str() + .with_context(|| { + format!( + "Failed to prase cache path '{}' as utf-8 string", + cache_path.display() + ) + })? + .to_string(), + ), + is_focused_to_value(is_focused), + ) + } else { + (None, None) + }; + + let duration: Option<f64> = video.duration.as_secs_f64(); + let last_status_change: i64 = video.last_status_change.as_secs(); + let publish_date: Option<i64> = video.publish_date.map(|pd| pd.as_secs()); + let watch_progress: i64 = + i64::try_from(video.watch_progress.as_secs()).expect("This should never exceed a u32"); + + let mut tx = app.database.begin().await?; + query!( + r#" + INSERT INTO videos ( + description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + title, + url, + watch_progress, + cache_path + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + "#, + video.description, + duration, + extractor_hash, + is_focused, + last_status_change, + parent_subscription_name, + publish_date, + status, + thumbnail_url, + video.title, + url, + watch_progress, + cache_path, + ) + .execute(&mut *tx) + .await?; + + query!( + r#" + INSERT INTO video_options ( + extractor_hash, + subtitle_langs, + playback_speed) + VALUES (?, ?, ?); + "#, + extractor_hash, + default_subtitle_langs, + default_mpv_playback_speed + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/video_database/set/playlist.rs b/crates/yt/src/storage/video_database/set/playlist.rs new file mode 100644 index 0000000..547df21 --- /dev/null +++ b/crates/yt/src/storage/video_database/set/playlist.rs @@ -0,0 +1,101 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use log::debug; +use sqlx::query; + +use crate::{ + app::App, + storage::video_database::{extractor_hash::ExtractorHash, get}, +}; + +/// Set a video to be focused. +/// This optionally takes another video hash, which marks the old focused video. +/// This will then be disabled. +/// +/// # Panics +/// Only if internal assertions fail. +pub async fn focused( + app: &App, + new_video_hash: &ExtractorHash, + old_video_hash: Option<&ExtractorHash>, +) -> Result<()> { + unfocused(app, old_video_hash).await?; + + debug!("Focusing video: '{new_video_hash}'"); + let new_hash = new_video_hash.hash().to_string(); + query!( + r#" + UPDATE videos + SET is_focused = 1 + WHERE extractor_hash = ?; + "#, + new_hash, + ) + .execute(&app.database) + .await?; + + assert_eq!( + *new_video_hash, + get::currently_focused_video(app) + .await? + .expect("This is some at this point") + .extractor_hash + ); + Ok(()) +} + +/// Set a video to be no longer focused. +/// This will use the supplied `video_hash` if it is [`Some`], otherwise it will simply un-focus +/// the currently focused video. +/// +/// # Panics +/// Only if internal assertions fail. +pub async fn unfocused(app: &App, video_hash: Option<&ExtractorHash>) -> Result<()> { + let hash = if let Some(hash) = video_hash { + hash.hash().to_string() + } else { + let output = query!( + r#" + SELECT extractor_hash + FROM videos + WHERE is_focused = 1; + "#, + ) + .fetch_optional(&app.database) + .await?; + + if let Some(output) = output { + output.extractor_hash + } else { + // There is no unfocused video right now. + return Ok(()); + } + }; + debug!("Unfocusing video: '{hash}'"); + + query!( + r#" + UPDATE videos + SET is_focused = NULL + WHERE extractor_hash = ?; + "#, + hash + ) + .execute(&app.database) + .await?; + + assert!( + get::currently_focused_video(app).await?.is_none(), + "We assumed that the video we just removed was actually a focused one." + ); + Ok(()) +} diff --git a/crates/yt/src/subscribe/mod.rs b/crates/yt/src/subscribe/mod.rs new file mode 100644 index 0000000..7ac0be4 --- /dev/null +++ b/crates/yt/src/subscribe/mod.rs @@ -0,0 +1,184 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::str::FromStr; + +use anyhow::{Context, Result, bail}; +use futures::FutureExt; +use log::warn; +use tokio::io::{AsyncBufRead, AsyncBufReadExt}; +use url::Url; +use yt_dlp::{YoutubeDLOptions, json_get}; + +use crate::{ + app::App, + storage::subscriptions::{ + Subscription, add_subscription, check_url, get, remove_all, remove_subscription, + }, + unreachable::Unreachable, +}; + +pub async fn unsubscribe(app: &App, name: String) -> Result<()> { + let present_subscriptions = get(app).await?; + + if let Some(subscription) = present_subscriptions.0.get(&name) { + remove_subscription(app, subscription).await?; + } else { + bail!("Couldn't find subscription: '{}'", &name); + } + + Ok(()) +} + +pub async fn import<W: AsyncBufRead + AsyncBufReadExt + Unpin>( + app: &App, + reader: W, + force: bool, +) -> Result<()> { + if force { + remove_all(app).await?; + } + + let mut lines = reader.lines(); + while let Some(line) = lines.next_line().await? { + let url = + Url::from_str(&line).with_context(|| format!("Failed to parse '{line}' as url"))?; + match subscribe(app, None, url) + .await + .with_context(|| format!("Failed to subscribe to: '{line}'")) + { + Ok(()) => (), + Err(err) => eprintln!( + "Error while subscribing to '{}': '{}'", + line, + err.source().unreachable("Should have a source") + ), + } + } + + Ok(()) +} + +pub async fn subscribe(app: &App, name: Option<String>, url: Url) -> Result<()> { + if !(url.as_str().ends_with("videos") + || url.as_str().ends_with("streams") + || url.as_str().ends_with("shorts") + || url.as_str().ends_with("videos/") + || url.as_str().ends_with("streams/") + || url.as_str().ends_with("shorts/")) + && url.as_str().contains("youtube.com") + { + warn!( + "Your youtbe url does not seem like it actually tracks a channels playlist (videos, streams, shorts). Adding subscriptions for each of them..." + ); + + let url = Url::parse(&(url.as_str().to_owned() + "/")) + .unreachable("This was an url, it should stay one"); + + if let Some(name) = name { + let out: Result<()> = async move { + actual_subscribe( + app, + Some(name.clone() + " {Videos}"), + url.join("videos/") + .unreachable("The url should allow being joined onto"), + ) + .await + .with_context(|| { + format!("Failed to subscribe to '{}'", name.clone() + " {Videos}") + })?; + + actual_subscribe( + app, + Some(name.clone() + " {Streams}"), + url.join("streams/").unreachable("See above."), + ) + .await + .with_context(|| { + format!("Failed to subscribe to '{}'", name.clone() + " {Streams}") + })?; + + actual_subscribe( + app, + Some(name.clone() + " {Shorts}"), + url.join("shorts/").unreachable("See above."), + ) + .await + .with_context(|| format!("Failed to subscribe to '{}'", name + " {Shorts}"))?; + + Ok(()) + } + .boxed() + .await; + + out?; + } else { + actual_subscribe(app, None, url.join("videos/").unreachable("See above.")) + .await + .with_context(|| format!("Failed to subscribe to the '{}' variant", "{Videos}"))?; + + actual_subscribe(app, None, url.join("streams/").unreachable("See above.")) + .await + .with_context(|| format!("Failed to subscribe to the '{}' variant", "{Streams}"))?; + + actual_subscribe(app, None, url.join("shorts/").unreachable("See above.")) + .await + .with_context(|| format!("Failed to subscribe to the '{}' variant", "{Shorts}"))?; + } + } else { + actual_subscribe(app, name, url).await?; + } + + Ok(()) +} + +async fn actual_subscribe(app: &App, name: Option<String>, url: Url) -> Result<()> { + if !check_url(url.clone()).await? { + bail!("The url ('{}') does not represent a playlist!", &url) + } + + let name = if let Some(name) = name { + name + } else { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + if info.get("_type") == Some(&serde_json::Value::String("Playlist".to_owned())) { + json_get!(info, "title", as_str).to_owned() + } else { + bail!("The url ('{}') does not represent a playlist!", &url) + } + }; + + let present_subscriptions = get(app).await?; + + if let Some(subs) = present_subscriptions.0.get(&name) { + bail!( + "The subscription '{}' could not be added, \ + as another one with the same name ('{}') already exists. It links to the Url: '{}'", + name, + name, + subs.url + ); + } + + let sub = Subscription { name, url }; + + add_subscription(app, &sub).await?; + + Ok(()) +} diff --git a/crates/yt/src/unreachable.rs b/crates/yt/src/unreachable.rs new file mode 100644 index 0000000..436fbb6 --- /dev/null +++ b/crates/yt/src/unreachable.rs @@ -0,0 +1,50 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +// This has been taken from: https://gitlab.torproject.org/tpo/core/arti/-/issues/950 + +// The functions here should be annotated with `#[inline(always)]`. +#![allow(clippy::inline_always)] + +use std::fmt::Debug; + +/// Trait for something that can possibly be unwrapped, like a Result or Option. +/// It provides semantic information, that unwrapping here should never happen. +pub trait Unreachable<T> { + /// Like `expect()`, but does not trigger clippy. + /// + /// # Usage + /// + /// This method only exists so that we can use it without hitting + /// `clippy::missing_panics_docs`. Therefore, we should only use it + /// for situations where we are certain that the panic cannot occur + /// unless something else is very broken. Consider instead using + /// `expect()` and adding a `Panics` section to your function + /// documentation. + /// + /// # Panics + /// + /// Panics if this is not an object that can be unwrapped, such as + /// None or an Err. + fn unreachable(self, msg: &str) -> T; +} +impl<T> Unreachable<T> for Option<T> { + #[inline(always)] + fn unreachable(self, msg: &str) -> T { + self.expect(msg) + } +} +impl<T, E: Debug> Unreachable<T> for Result<T, E> { + #[inline(always)] + fn unreachable(self, msg: &str) -> T { + self.expect(msg) + } +} diff --git a/crates/yt/src/update/mod.rs b/crates/yt/src/update/mod.rs new file mode 100644 index 0000000..f0b1e2c --- /dev/null +++ b/crates/yt/src/update/mod.rs @@ -0,0 +1,203 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{str::FromStr, time::Duration}; + +use anyhow::{Context, Ok, Result}; +use chrono::{DateTime, Utc}; +use log::{info, warn}; +use url::Url; +use yt_dlp::{InfoJson, json_cast, json_get}; + +use crate::{ + app::App, + select::selection_file::duration::MaybeDuration, + storage::{ + subscriptions::{self, Subscription}, + video_database::{ + Priority, TimeStamp, Video, VideoStatus, extractor_hash::ExtractorHash, + get::get_all_hashes, set::add_video, + }, + }, +}; + +mod updater; +use updater::Updater; + +pub async fn update( + app: &App, + max_backlog: usize, + subscription_names_to_update: Vec<String>, +) -> Result<()> { + let subscriptions = subscriptions::get(app).await?; + + let urls: Vec<_> = if subscription_names_to_update.is_empty() { + subscriptions.0.values().collect() + } else { + subscriptions + .0 + .values() + .filter(|sub| { + if subscription_names_to_update.contains(&sub.name) { + true + } else { + info!( + "Not updating subscription '{}' as it was not specified", + sub.name + ); + false + } + }) + .collect() + }; + + // We can get away with not having to re-fetch the hashes every time, as the returned video + // should not contain duplicates. + let hashes = get_all_hashes(app).await?; + + { + let mut updater = Updater::new(max_backlog, &hashes); + updater.update(app, &urls).await?; + } + + Ok(()) +} + +#[allow(clippy::too_many_lines)] +pub fn video_entry_to_video(entry: &InfoJson, sub: Option<&Subscription>) -> Result<Video> { + fn fmt_context(date: &str, extended: Option<&str>) -> String { + let f = format!( + "Failed to parse the `upload_date` of the entry ('{date}'). \ + Expected `YYYY-MM-DD`, has the format changed?" + ); + if let Some(date_string) = extended { + format!("{f}\nThe parsed '{date_string}' can't be turned to a valid UTC date.'") + } else { + f + } + } + + let publish_date = if let Some(date) = &entry.get("upload_date") { + let date = json_cast!(date, as_str); + + let year: u32 = date + .chars() + .take(4) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let month: u32 = date + .chars() + .skip(4) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + let day: u32 = date + .chars() + .skip(4 + 2) + .take(2) + .collect::<String>() + .parse() + .with_context(|| fmt_context(date, None))?; + + let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z"); + Some( + DateTime::<Utc>::from_str(&date_string) + .with_context(|| fmt_context(date, Some(&date_string)))? + .timestamp(), + ) + } else { + warn!( + "The video '{}' lacks it's upload date!", + json_get!(entry, "title", as_str) + ); + None + }; + + let thumbnail_url = match (&entry.get("thumbnails"), &entry.get("thumbnail")) { + (None, None) => None, + (None, Some(thumbnail)) => Some(Url::from_str(json_cast!(thumbnail, as_str))?), + + // TODO: The algorithm is not exactly the best <2024-05-28> + (Some(thumbnails), None) => { + if let Some(thumbnail) = json_cast!(thumbnails, as_array).first() { + Some(Url::from_str(json_get!( + json_cast!(thumbnail, as_object), + "url", + as_str + ))?) + } else { + None + } + } + (Some(_), Some(thumnail)) => Some(Url::from_str(json_cast!(thumnail, as_str))?), + }; + + let url = { + let smug_url: Url = json_get!(entry, "webpage_url", as_str).parse()?; + // unsmuggle_url(&smug_url)? + smug_url + }; + + let extractor_hash = blake3::hash(json_get!(entry, "id", as_str).as_bytes()); + + let subscription_name = if let Some(sub) = sub { + Some(sub.name.clone()) + } else if let Some(uploader) = entry.get("uploader") { + if entry.get("webpage_url_domain") + == Some(&serde_json::Value::String("youtube.com".to_owned())) + { + Some(format!("{uploader} - Videos")) + } else { + Some(json_cast!(uploader, as_str).to_owned()) + } + } else { + None + }; + + let video = Video { + description: entry + .get("description") + .map(|val| json_cast!(val, as_str).to_owned()), + duration: MaybeDuration::from_maybe_secs_f64( + entry.get("duration").map(|val| json_cast!(val, as_f64)), + ), + extractor_hash: ExtractorHash::from_hash(extractor_hash), + last_status_change: TimeStamp::from_now(), + parent_subscription_name: subscription_name, + priority: Priority::default(), + publish_date: publish_date.map(TimeStamp::from_secs), + status: VideoStatus::Pick, + thumbnail_url, + title: json_get!(entry, "title", as_str).to_owned(), + url, + watch_progress: Duration::default(), + }; + Ok(video) +} + +async fn process_subscription(app: &App, sub: &Subscription, entry: InfoJson) -> Result<()> { + let video = + video_entry_to_video(&entry, Some(sub)).context("Failed to parse search entry as Video")?; + + add_video(app, video.clone()) + .await + .with_context(|| format!("Failed to add video to database: '{}'", video.title))?; + println!( + "{}", + &video + .to_line_display(app) + .await + .with_context(|| format!("Failed to format video: '{}'", video.title))? + ); + Ok(()) +} diff --git a/crates/yt/src/update/updater.rs b/crates/yt/src/update/updater.rs new file mode 100644 index 0000000..8da654b --- /dev/null +++ b/crates/yt/src/update/updater.rs @@ -0,0 +1,167 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::io::{Write, stderr}; + +use anyhow::{Context, Result}; +use blake3::Hash; +use futures::{ + StreamExt, TryStreamExt, + stream::{self}, +}; +use log::{Level, debug, error, log_enabled}; +use serde_json::json; +use yt_dlp::{InfoJson, YoutubeDLOptions, json_cast, json_get}; + +use crate::{ + ansi_escape_codes::{clear_whole_line, move_to_col}, + app::App, + storage::subscriptions::Subscription, +}; + +use super::process_subscription; + +pub(super) struct Updater<'a> { + max_backlog: usize, + hashes: &'a [Hash], +} + +impl<'a> Updater<'a> { + pub(super) fn new(max_backlog: usize, hashes: &'a [Hash]) -> Self { + Self { + max_backlog, + hashes, + } + } + + pub(super) async fn update( + &mut self, + app: &App, + subscriptions: &[&Subscription], + ) -> Result<()> { + let mut stream = stream::iter(subscriptions) + .map(|sub| self.get_new_entries(sub)) + .buffer_unordered(100); + + while let Some(output) = stream.next().await { + let mut entries = output?; + + if entries.is_empty() { + continue; + } + + let (sub, entry) = entries.remove(0); + process_subscription(app, sub, entry).await?; + + let entry_stream: Result<()> = stream::iter(entries) + .map(|(sub, entry)| process_subscription(app, sub, entry)) + .buffer_unordered(100) + .try_collect() + .await; + entry_stream?; + } + + Ok(()) + } + + async fn get_new_entries( + &self, + sub: &'a Subscription, + ) -> Result<Vec<(&'a Subscription, InfoJson)>> { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", self.max_backlog) + .set("noplaylist", false) + .set( + "extractor_args", + json! {{"youtubetab": {"approximate_date": [""]}}}, + ) + // TODO: This also removes unlisted and other stuff. Find a good way to remove the + // members-only videos from the feed. <2025-04-17> + .set("match-filter", "availability=public") + .build()?; + + if !log_enabled!(Level::Debug) { + clear_whole_line(); + move_to_col(1); + eprint!("Checking playlist {}...", sub.name); + move_to_col(1); + stderr().flush()?; + } + + let info = yt_dlp + .extract_info(&sub.url, false, false) + .with_context(|| format!("Failed to get playlist '{}'.", sub.name))?; + + let empty = vec![]; + let entries = info + .get("entries") + .map_or(&empty, |val| json_cast!(val, as_array)); + + let valid_entries: Vec<(&Subscription, InfoJson)> = entries + .iter() + .take(self.max_backlog) + .filter_map(|entry| -> Option<(&Subscription, InfoJson)> { + let id = json_get!(entry, "id", as_str); + let extractor_hash = blake3::hash(id.as_bytes()); + if self.hashes.contains(&extractor_hash) { + debug!("Skipping entry, as it is already present: '{extractor_hash}'",); + None + } else { + Some((sub, json_cast!(entry, as_object).to_owned())) + } + }) + .collect(); + + let processed_entries: Vec<(&Subscription, InfoJson)> = stream::iter(valid_entries) + .map( + async |(sub, entry)| match yt_dlp.process_ie_result(entry, false) { + Ok(output) => Ok((sub, output)), + Err(err) => Err(err), + }, + ) + .buffer_unordered(100) + .collect::<Vec<_>>() + .await + .into_iter() + // Don't fail the whole update, if one of the entries fails to fetch. + .filter_map(|base| match base { + Ok(ok) => Some(ok), + Err(err) => { + // TODO(@bpeetz): Add this <2025-06-13> + // if let YtDlpError::PythonError { error, kind } = &err { + // if kind.as_str() == "<class 'yt_dlp.utils.DownloadError'>" + // && error.to_string().as_str().contains( + // "Join this channel to get access to members-only content ", + // ) + // { + // // Hide this error + // } else { + // let error_string = error.to_string(); + // let error = error_string + // .strip_prefix("DownloadError: \u{1b}[0;31mERROR:\u{1b}[0m ") + // .expect("This prefix should exists"); + // error!("{error}"); + // } + // return None; + // } + + // TODO(@bpeetz): Ideally, we _would_ actually exit on unexpected errors, but + // this is fine for now. <2025-06-13> + // Some(Err(err).context("Failed to process new entries.")) + error!("While processing entry: {err}"); + None + } + }) + .collect(); + + Ok(processed_entries) + } +} diff --git a/crates/yt/src/version/mod.rs b/crates/yt/src/version/mod.rs new file mode 100644 index 0000000..05d85e0 --- /dev/null +++ b/crates/yt/src/version/mod.rs @@ -0,0 +1,63 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::process::Command; + +use anyhow::{Context, Result}; +use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; + +use crate::{config::Config, storage::migrate::get_version_db}; + +fn get_cmd_version(cmd: &str) -> Result<String> { + let out = String::from_utf8( + Command::new(cmd) + .arg("--version") + .output() + .with_context(|| format!("Failed to run `{cmd} --version`"))? + .stdout, + ) + .context("Failed to interpret output as utf8")?; + + Ok(out.trim().to_owned()) +} + +pub async fn show(config: &Config) -> Result<()> { + let db_version = { + let options = SqliteConnectOptions::new() + .filename(&config.paths.database_path) + .optimize_on_close(true, None) + .create_if_missing(true); + + let pool = SqlitePool::connect_with(options) + .await + .context("Failed to connect to database!")?; + + get_version_db(&pool) + .await + .context("Failed to determine database version")? + }; + + // TODO(@bpeetz): Use `pyo3`'s build in mechanism instead of executing the python CLI <2025-02-21> + let python_version = get_cmd_version("python")?; + let yt_dlp_version = get_cmd_version("yt-dlp")?; + + println!( + "{}: {} + +db version: {db_version} + +python: {python_version} +yt-dlp: {yt_dlp_version}", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_VERSION"), + ); + + Ok(()) +} diff --git a/crates/yt/src/videos/display/format_video.rs b/crates/yt/src/videos/display/format_video.rs new file mode 100644 index 0000000..b97acb1 --- /dev/null +++ b/crates/yt/src/videos/display/format_video.rs @@ -0,0 +1,94 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; + +use crate::{app::App, comments::output::format_text, storage::video_database::Video}; + +impl Video { + pub async fn to_info_display(&self, app: &App) -> Result<String> { + let cache_path = self.cache_path_fmt(app); + let description = self.description_fmt(); + let duration = self.duration_fmt(app); + let extractor_hash = self.extractor_hash_fmt(app).await?; + let in_playlist = self.in_playlist_fmt(app); + let last_status_change = self.last_status_change_fmt(app); + let parent_subscription_name = self.parent_subscription_name_fmt(app); + let priority = self.priority_fmt(); + let publish_date = self.publish_date_fmt(app); + let status = self.status_fmt(app); + let thumbnail_url = self.thumbnail_url_fmt(); + let title = self.title_fmt(app); + let url = self.url_fmt(app); + let watch_progress = self.watch_progress_fmt(app); + let video_options = self.video_options_fmt(app).await?; + + let watched_percentage_fmt = { + if let Some(duration) = self.duration.as_secs() { + format!( + " (watched: {:0.0}%)", + (self.watch_progress.as_secs() / duration) * 100 + ) + } else { + format!(" {watch_progress}") + } + }; + + let string = format!( + "\ +{title} ({extractor_hash}) +| -> {cache_path} +| -> {duration}{watched_percentage_fmt} +| -> {parent_subscription_name} +| -> priority: {priority} +| -> {publish_date} +| -> status: {status} since {last_status_change} ({in_playlist}) +| -> {thumbnail_url} +| -> {url} +| -> options: {} +{}\n", + video_options.to_string().trim(), + format_text(description.to_string().as_str()) + ); + Ok(string) + } + + pub async fn to_line_display(&self, app: &App) -> Result<String> { + let f = format!( + "{} {} {} {} {} {}", + self.status_fmt(app), + self.extractor_hash_fmt(app).await?, + self.title_fmt(app), + self.publish_date_fmt(app), + self.parent_subscription_name_fmt(app), + self.duration_fmt(app) + ); + + Ok(f) + } + + pub async fn to_select_file_display(&self, app: &App) -> Result<String> { + let f = format!( + r#"{}{} {} "{}" "{}" "{}" "{}" "{}"{}"#, + self.status_fmt_no_color(), + self.video_options_fmt_no_color(app).await?, + self.extractor_hash_fmt_no_color(app).await?, + self.title_fmt_no_color(), + self.publish_date_fmt_no_color(), + self.parent_subscription_name_fmt_no_color(), + self.duration_fmt_no_color(), + self.url_fmt_no_color(), + '\n' + ); + + Ok(f) + } +} diff --git a/crates/yt/src/videos/display/mod.rs b/crates/yt/src/videos/display/mod.rs new file mode 100644 index 0000000..1188569 --- /dev/null +++ b/crates/yt/src/videos/display/mod.rs @@ -0,0 +1,229 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use owo_colors::OwoColorize; +use url::Url; + +use crate::{ + app::App, + select::selection_file::duration::MaybeDuration, + storage::video_database::{TimeStamp, Video, VideoStatus, get::get_video_opts}, +}; + +use anyhow::{Context, Result}; + +pub mod format_video; + +macro_rules! get { + ($value:expr, $key:ident, $name:expr, $code:tt) => { + if let Some(value) = &$value.$key { + $code(value) + } else { + concat!("[No ", $name, "]").to_owned() + } + }; +} + +fn maybe_add_color<F>(app: &App, input: String, mut color_fn: F) -> String +where + F: FnMut(String) -> String, +{ + if app.config.global.display_colors { + color_fn(input) + } else { + input + } +} +impl Video { + #[must_use] + pub fn cache_path_fmt(&self, app: &App) -> String { + let cache_path = if let VideoStatus::Cached { + cache_path, + is_focused: _, + } = &self.status + { + cache_path.to_string_lossy().to_string() + } else { + "[No Cache Path]".to_owned() + }; + maybe_add_color(app, cache_path, |v| v.blue().bold().to_string()) + } + + #[must_use] + pub fn description_fmt(&self) -> String { + get!( + self, + description, + "Description", + (|value: &str| value.to_owned()) + ) + } + + #[must_use] + pub fn duration_fmt_no_color(&self) -> String { + self.duration.to_string() + } + #[must_use] + pub fn duration_fmt(&self, app: &App) -> String { + let duration = self.duration_fmt_no_color(); + maybe_add_color(app, duration, |v| v.cyan().bold().to_string()) + } + + #[must_use] + pub fn watch_progress_fmt(&self, app: &App) -> String { + maybe_add_color( + app, + MaybeDuration::from_std(self.watch_progress).to_string(), + |v| v.cyan().bold().to_string(), + ) + } + + pub async fn extractor_hash_fmt_no_color(&self, app: &App) -> Result<String> { + let hash = self + .extractor_hash + .into_short_hash(app) + .await + .with_context(|| { + format!( + "Failed to format extractor hash, whilst formatting video: '{}'", + self.title + ) + })? + .to_string(); + Ok(hash) + } + pub async fn extractor_hash_fmt(&self, app: &App) -> Result<String> { + let hash = self.extractor_hash_fmt_no_color(app).await?; + Ok(maybe_add_color(app, hash, |v| { + v.bright_purple().italic().to_string() + })) + } + + #[must_use] + pub fn in_playlist_fmt(&self, app: &App) -> String { + let output = match &self.status { + VideoStatus::Pick + | VideoStatus::Watch + | VideoStatus::Watched + | VideoStatus::Drop + | VideoStatus::Dropped => "Not in the playlist", + VideoStatus::Cached { is_focused, .. } => { + if *is_focused { + "In the playlist and focused" + } else { + "In the playlist" + } + } + }; + maybe_add_color(app, output.to_owned(), |v| v.yellow().italic().to_string()) + } + #[must_use] + pub fn last_status_change_fmt(&self, app: &App) -> String { + maybe_add_color(app, self.last_status_change.to_string(), |v| { + v.bright_cyan().to_string() + }) + } + + #[must_use] + pub fn parent_subscription_name_fmt_no_color(&self) -> String { + get!( + self, + parent_subscription_name, + "author", + (|sub: &str| sub.replace('"', "'")) + ) + } + #[must_use] + pub fn parent_subscription_name_fmt(&self, app: &App) -> String { + let psn = self.parent_subscription_name_fmt_no_color(); + maybe_add_color(app, psn, |v| v.bright_magenta().to_string()) + } + + #[must_use] + pub fn priority_fmt(&self) -> String { + self.priority.to_string() + } + + #[must_use] + pub fn publish_date_fmt_no_color(&self) -> String { + get!( + self, + publish_date, + "release date", + (|date: &TimeStamp| date.to_string()) + ) + } + #[must_use] + pub fn publish_date_fmt(&self, app: &App) -> String { + let date = self.publish_date_fmt_no_color(); + maybe_add_color(app, date, |v| v.bright_white().bold().to_string()) + } + + #[must_use] + pub fn status_fmt_no_color(&self) -> String { + // TODO: We might support `.trim()`ing that, as the extra whitespace could be bad in the + // selection file. <2024-10-07> + self.status.as_marker().as_command().to_string() + } + #[must_use] + pub fn status_fmt(&self, app: &App) -> String { + let status = self.status_fmt_no_color(); + maybe_add_color(app, status, |v| v.red().bold().to_string()) + } + + #[must_use] + pub fn thumbnail_url_fmt(&self) -> String { + get!( + self, + thumbnail_url, + "thumbnail URL", + (|url: &Url| url.to_string()) + ) + } + + #[must_use] + pub fn title_fmt_no_color(&self) -> String { + self.title.replace(['"', '„', '”', '“'], "'") + } + #[must_use] + pub fn title_fmt(&self, app: &App) -> String { + let title = self.title_fmt_no_color(); + maybe_add_color(app, title, |v| v.green().bold().to_string()) + } + + #[must_use] + pub fn url_fmt_no_color(&self) -> String { + self.url.as_str().replace('"', "\\\"") + } + #[must_use] + pub fn url_fmt(&self, app: &App) -> String { + let url = self.url_fmt_no_color(); + maybe_add_color(app, url, |v| v.italic().to_string()) + } + + pub async fn video_options_fmt_no_color(&self, app: &App) -> Result<String> { + let video_options = { + let opts = get_video_opts(app, &self.extractor_hash) + .await + .with_context(|| { + format!("Failed to get video options for video: '{}'", self.title) + })? + .to_cli_flags(app); + let opts_white = if opts.is_empty() { "" } else { " " }; + format!("{opts_white}{opts}") + }; + Ok(video_options) + } + pub async fn video_options_fmt(&self, app: &App) -> Result<String> { + let opts = self.video_options_fmt_no_color(app).await?; + Ok(maybe_add_color(app, opts, |v| v.bright_green().to_string())) + } +} diff --git a/crates/yt/src/videos/mod.rs b/crates/yt/src/videos/mod.rs new file mode 100644 index 0000000..e821772 --- /dev/null +++ b/crates/yt/src/videos/mod.rs @@ -0,0 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use anyhow::Result; +use futures::{TryStreamExt, stream::FuturesUnordered}; +use nucleo_matcher::{ + Matcher, + pattern::{CaseMatching, Normalization, Pattern}, +}; + +pub mod display; + +use crate::{ + app::App, + storage::video_database::{Video, VideoStatusMarker, get}, +}; + +async fn to_line_display_owned(video: Video, app: &App) -> Result<String> { + video.to_line_display(app).await +} + +pub async fn query(app: &App, limit: Option<usize>, search_query: Option<String>) -> Result<()> { + let all_videos = get::videos(app, VideoStatusMarker::ALL).await?; + + // turn one video to a color display, to pre-warm the hash shrinking cache + if let Some(val) = all_videos.first() { + val.to_line_display(app).await?; + } + + let limit = limit.unwrap_or(all_videos.len()); + + let all_video_strings: Vec<String> = all_videos + .into_iter() + .take(limit) + .map(|vid| to_line_display_owned(vid, app)) + .collect::<FuturesUnordered<_>>() + .try_collect::<Vec<String>>() + .await?; + + if let Some(query) = search_query { + let mut matcher = Matcher::new(nucleo_matcher::Config::DEFAULT.match_paths()); + + let pattern_matches = Pattern::parse( + &query.replace(' ', "\\ "), + CaseMatching::Ignore, + Normalization::Smart, + ) + .match_list(all_video_strings, &mut matcher); + + pattern_matches + .iter() + .rev() + .for_each(|(val, key)| println!("{val} ({key})")); + } else { + println!("{}", all_video_strings.join("\n")); + } + + Ok(()) +} diff --git a/crates/yt/src/watch/mod.rs b/crates/yt/src/watch/mod.rs new file mode 100644 index 0000000..c32a76f --- /dev/null +++ b/crates/yt/src/watch/mod.rs @@ -0,0 +1,178 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use anyhow::{Context, Result}; +use libmpv2::{Mpv, events::EventContext}; +use log::{debug, info, trace, warn}; +use playlist_handler::{reload_mpv_playlist, save_watch_progress}; +use tokio::{task, time::sleep}; + +use self::playlist_handler::Status; +use crate::{ + app::App, + cache::maintain, + storage::video_database::{get, notify::wait_for_db_write}, +}; + +pub mod playlist; +pub mod playlist_handler; + +fn init_mpv(app: &App) -> Result<(Mpv, EventContext)> { + // set some default values, to make things easier (these can be overridden by the config file, + // which we load later) + let mpv = Mpv::with_initializer(|mpv| { + // Enable default key bindings, so the user can actually interact with + // the player (and e.g. close the window). + mpv.set_property("input-default-bindings", "yes")?; + mpv.set_property("input-vo-keyboard", "yes")?; + + // Show the on screen controller. + mpv.set_property("osc", "yes")?; + + // Don't automatically advance to the next video (or exit the player) + mpv.set_option("keep-open", "always")?; + + // Always display an window, even for non-video playback. + // As mpv does not have cli access, no window means no control and no user feedback. + mpv.set_option("force-window", "yes")?; + Ok(()) + }) + .context("Failed to initialize mpv")?; + + let config_path = &app.config.paths.mpv_config_path; + if config_path.try_exists()? { + info!("Found mpv.conf at '{}'!", config_path.display()); + mpv.command( + "load-config-file", + &[config_path + .to_str() + .context("Failed to parse the config path is utf8-stringt")?], + )?; + } else { + warn!( + "Did not find a mpv.conf file at '{}'", + config_path.display() + ); + } + + let input_path = &app.config.paths.mpv_input_path; + if input_path.try_exists()? { + info!("Found mpv.input.conf at '{}'!", input_path.display()); + mpv.command( + "load-input-conf", + &[input_path + .to_str() + .context("Failed to parse the input path as utf8 string")?], + )?; + } else { + warn!( + "Did not find a mpv.input.conf file at '{}'", + input_path.display() + ); + } + + let ev_ctx = EventContext::new(mpv.ctx); + ev_ctx.disable_deprecated_events()?; + + Ok((mpv, ev_ctx)) +} + +pub async fn watch(app: Arc<App>) -> Result<()> { + maintain(&app, false).await?; + + let (mpv, mut ev_ctx) = init_mpv(&app).context("Failed to initialize mpv instance")?; + let mpv = Arc::new(mpv); + reload_mpv_playlist(&app, &mpv, None, None).await?; + + let should_break = Arc::new(AtomicBool::new(false)); + + let local_app = Arc::clone(&app); + let local_mpv = Arc::clone(&mpv); + let local_should_break = Arc::clone(&should_break); + let progress_handle = task::spawn(async move { + loop { + if local_should_break.load(Ordering::Relaxed) { + break; + } + + if get::currently_focused_video(&local_app).await?.is_some() { + save_watch_progress(&local_app, &local_mpv).await?; + } + + sleep(Duration::from_secs(30)).await; + } + + Ok::<(), anyhow::Error>(()) + }); + + let mut have_warned = (false, 0); + 'watchloop: loop { + 'waitloop: while let Ok(value) = playlist_handler::status(&app).await { + match value { + Status::NoMoreAvailable => { + break 'watchloop; + } + Status::NoCached { marked_watch } => { + // try again next time. + if have_warned.0 { + if have_warned.1 != marked_watch { + warn!("Now {} videos are marked as to be watched.", marked_watch); + have_warned.1 = marked_watch; + } + } else { + warn!( + "There is nothing to watch yet, but still {} videos marked as to be watched. \ + Will idle, until they become available", + marked_watch + ); + have_warned = (true, marked_watch); + } + wait_for_db_write(&app).await?; + } + Status::Available { newly_available } => { + debug!("Check and found {newly_available} videos!"); + have_warned.0 = false; + + // Something just became available! + break 'waitloop; + } + } + } + + if let Some(ev) = ev_ctx.wait_event(30.) { + match ev { + Ok(event) => { + trace!("Mpv event triggered: {:#?}", event); + if playlist_handler::handle_mpv_event(&app, &mpv, &event) + .await + .with_context(|| format!("Failed to handle mpv event: '{event:#?}'"))? + { + break; + } + } + Err(e) => debug!("Mpv Event errored: {}", e), + } + } + } + + should_break.store(true, Ordering::Relaxed); + progress_handle.await??; + + Ok(()) +} diff --git a/crates/yt/src/watch/playlist.rs b/crates/yt/src/watch/playlist.rs new file mode 100644 index 0000000..ff383d0 --- /dev/null +++ b/crates/yt/src/watch/playlist.rs @@ -0,0 +1,99 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::path::Path; + +use crate::{ + ansi_escape_codes::{cursor_up, erase_in_display_from_cursor}, + app::App, + storage::video_database::{Video, VideoStatus, get, notify::wait_for_db_write}, +}; + +use anyhow::Result; +use futures::{TryStreamExt, stream::FuturesOrdered}; + +/// Extract the values of the [`VideoStatus::Cached`] value from a Video. +fn cache_values(video: &Video) -> (&Path, bool) { + if let VideoStatus::Cached { + cache_path, + is_focused, + } = &video.status + { + (cache_path, *is_focused) + } else { + unreachable!("All of these videos should be cached"); + } +} + +/// # Panics +/// Only if internal assertions fail. +pub async fn playlist(app: &App, watch: bool) -> Result<()> { + let mut previous_output_length = 0; + loop { + let playlist = get::playlist(app).await?.to_videos(); + + let output = playlist + .into_iter() + .map(|video| async move { + let mut output = String::new(); + + let (_, is_focused) = cache_values(&video); + + if is_focused { + output.push_str("🔻 "); + } else { + output.push_str(" "); + } + + output.push_str(&video.title_fmt(app)); + + output.push_str(" ("); + output.push_str(&video.parent_subscription_name_fmt(app)); + output.push(')'); + + output.push_str(" ["); + output.push_str(&video.duration_fmt(app)); + + if is_focused { + output.push_str(" ("); + output.push_str(&if let Some(duration) = video.duration.as_secs() { + format!("{:0.0}%", (video.watch_progress.as_secs() / duration) * 100) + } else { + video.watch_progress_fmt(app) + }); + output.push(')'); + } + output.push(']'); + + output.push('\n'); + + Ok::<String, anyhow::Error>(output) + }) + .collect::<FuturesOrdered<_>>() + .try_collect::<String>() + .await?; + + // Delete the previous output + cursor_up(previous_output_length); + erase_in_display_from_cursor(); + + previous_output_length = output.chars().filter(|ch| *ch == '\n').count(); + + print!("{output}"); + + if !watch { + break; + } + + wait_for_db_write(app).await?; + } + + Ok(()) +} diff --git a/crates/yt/src/watch/playlist_handler/client_messages/mod.rs b/crates/yt/src/watch/playlist_handler/client_messages/mod.rs new file mode 100644 index 0000000..6f7a59e --- /dev/null +++ b/crates/yt/src/watch/playlist_handler/client_messages/mod.rs @@ -0,0 +1,98 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{env, time::Duration}; + +use crate::{app::App, comments}; + +use anyhow::{Context, Result, bail}; +use libmpv2::Mpv; +use tokio::process::Command; + +use super::mpv_message; + +async fn run_self_in_external_command(app: &App, args: &[&str]) -> Result<()> { + let binary = + env::current_exe().context("Failed to determine the current executable to re-execute")?; + + let status = Command::new("riverctl") + .args(["focus-output", "next"]) + .status() + .await?; + if !status.success() { + bail!("focusing the next output failed!"); + } + + let arguments = [ + &[ + "--title", + "floating please", + "--command", + binary + .to_str() + .context("Failed to turn the executable path to a utf8-string")?, + "--db-path", + app.config + .paths + .database_path + .to_str() + .context("Failed to parse the database_path as a utf8-string")?, + ], + args, + ] + .concat(); + + let status = Command::new("alacritty").args(arguments).status().await?; + if !status.success() { + bail!("Falied to start `yt comments`"); + } + + let status = Command::new("riverctl") + .args(["focus-output", "next"]) + .status() + .await?; + + if !status.success() { + bail!("focusing the next output failed!"); + } + + Ok(()) +} + +pub(super) async fn handle_yt_description_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["description"]).await?; + Ok(()) +} +pub(super) async fn handle_yt_description_local(app: &App, mpv: &Mpv) -> Result<()> { + let description: String = comments::description::get(app) + .await? + .chars() + .take(app.config.watch.local_displays_length) + .collect(); + + mpv_message(mpv, &description, Duration::from_secs(6))?; + Ok(()) +} + +pub(super) async fn handle_yt_comments_external(app: &App) -> Result<()> { + run_self_in_external_command(app, &["comments"]).await?; + Ok(()) +} +pub(super) async fn handle_yt_comments_local(app: &App, mpv: &Mpv) -> Result<()> { + let comments: String = comments::get(app) + .await? + .render(false) + .chars() + .take(app.config.watch.local_displays_length) + .collect(); + + mpv_message(mpv, &comments, Duration::from_secs(6))?; + Ok(()) +} diff --git a/crates/yt/src/watch/playlist_handler/mod.rs b/crates/yt/src/watch/playlist_handler/mod.rs new file mode 100644 index 0000000..29b8f39 --- /dev/null +++ b/crates/yt/src/watch/playlist_handler/mod.rs @@ -0,0 +1,342 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{cmp::Ordering, time::Duration}; + +use crate::{ + app::App, + storage::video_database::{ + VideoStatus, VideoStatusMarker, + extractor_hash::ExtractorHash, + get::{self, Playlist, PlaylistIndex}, + set, + }, +}; + +use anyhow::{Context, Result}; +use libmpv2::{EndFileReason, Mpv, events::Event}; +use log::{debug, info}; + +mod client_messages; + +#[derive(Debug, Clone, Copy)] +pub enum Status { + /// There are no videos cached and no more marked to be watched. + /// Waiting is pointless. + NoMoreAvailable, + + /// There are no videos cached, but some (> 0) are marked to be watched. + /// So we should wait for them to become available. + NoCached { marked_watch: usize }, + + /// There are videos cached and ready to be inserted into the playback queue. + Available { newly_available: usize }, +} + +fn mpv_message(mpv: &Mpv, message: &str, time: Duration) -> Result<()> { + mpv.command( + "show-text", + &[message, time.as_millis().to_string().as_str()], + )?; + Ok(()) +} + +async fn apply_video_options(app: &App, mpv: &Mpv, video: &ExtractorHash) -> Result<()> { + let options = get::video_mpv_opts(app, video).await?; + let video = get::video_by_hash(app, video).await?; + + mpv.set_property("speed", options.playback_speed)?; + + // We already start at 0, so setting it twice adds a uncomfortable skip sound. + if video.watch_progress.as_secs() != 0 { + mpv.set_property( + "time-pos", + i64::try_from(video.watch_progress.as_secs()).expect("This should not overflow"), + )?; + } + Ok(()) +} + +async fn mark_video_watched(app: &App, mpv: &Mpv) -> Result<()> { + let current_video = get::currently_focused_video(app) + .await? + .expect("This should be some at this point"); + + debug!( + "playlist handler will mark video '{}' watched.", + current_video.title + ); + + save_watch_progress(app, mpv).await?; + + set::video_watched(app, ¤t_video.extractor_hash).await?; + + Ok(()) +} + +/// Saves the `watch_progress` of the currently focused video. +pub(super) async fn save_watch_progress(app: &App, mpv: &Mpv) -> Result<()> { + let current_video = get::currently_focused_video(app) + .await? + .expect("This should be some at this point"); + let watch_progress = u32::try_from( + mpv.get_property::<i64>("time-pos") + .context("Failed to get the watchprogress of the currently playling video")?, + ) + .expect("This conversion should never fail as the `time-pos` property is positive"); + + debug!( + "Setting the watch progress for the current_video '{}' to {watch_progress}s", + current_video.title_fmt_no_color() + ); + + set::video_watch_progress(app, ¤t_video.extractor_hash, watch_progress).await +} + +/// Sync the mpv playlist with the internal playlist. +/// +/// This takes an `maybe_playlist` argument, if you have already fetched the playlist and want to +/// add that. +pub(super) async fn reload_mpv_playlist( + app: &App, + mpv: &Mpv, + maybe_playlist: Option<Playlist>, + maybe_index: Option<PlaylistIndex>, +) -> Result<()> { + fn get_playlist_count(mpv: &Mpv) -> Result<usize> { + mpv.get_property::<i64>("playlist/count") + .context("Failed to get mpv playlist len") + .map(|count| { + usize::try_from(count).expect("The playlist_count should always be positive") + }) + } + + if get_playlist_count(mpv)? != 0 { + // We could also use `loadlist`, but that would require use to start a unix socket or even + // write all the video paths to a file beforehand + mpv.command("playlist-clear", &[])?; + mpv.command("playlist-remove", &["current"])?; + } + + assert_eq!( + get_playlist_count(mpv)?, + 0, + "The playlist should be empty at this point." + ); + + let playlist = if let Some(p) = maybe_playlist { + p + } else { + get::playlist(app).await? + }; + + debug!("Will add {} videos to playlist.", playlist.len()); + playlist.into_iter().try_for_each(|cache_path| { + mpv.command( + "loadfile", + &[ + cache_path.to_str().with_context(|| { + format!( + "Failed to parse the video cache path ('{}') as valid utf8", + cache_path.display() + ) + })?, + "append-play", + ], + )?; + + Ok::<(), anyhow::Error>(()) + })?; + + let index = if let Some(index) = maybe_index { + let index = usize::from(index); + let playlist_length = get_playlist_count(mpv)?; + + match index.cmp(&playlist_length) { + Ordering::Greater => { + unreachable!( + "The index '{index}' execeeds the playlist length '{playlist_length}'." + ); + } + Ordering::Less => index, + Ordering::Equal => { + // The index is pointing to the end of the playlist. We could either go the second + // to last entry (i.e., one entry back) or wrap around to the start. + // We wrap around: + 0 + } + } + } else { + get::current_playlist_index(app) + .await? + .map_or(0, usize::from) + }; + mpv.set_property("playlist-pos", index.to_string().as_str())?; + + Ok(()) +} + +/// Return the status of the playback queue +pub async fn status(app: &App) -> Result<Status> { + let playlist = get::playlist(app).await?; + + let playlist_len = playlist.len(); + let marked_watch_num = get::videos(app, &[VideoStatusMarker::Watch]).await?.len(); + + if playlist_len == 0 && marked_watch_num == 0 { + Ok(Status::NoMoreAvailable) + } else if playlist_len == 0 && marked_watch_num != 0 { + Ok(Status::NoCached { + marked_watch: marked_watch_num, + }) + } else if playlist_len != 0 { + Ok(Status::Available { + newly_available: playlist_len, + }) + } else { + unreachable!( + "The playlist length is {playlist_len}, but the number of marked watch videos is {marked_watch_num}! This is a bug." + ); + } +} + +/// # Returns +/// This will return [`true`], if the event handling should be stopped +/// +/// # Panics +/// Only if internal assertions fail. +#[allow(clippy::too_many_lines)] +pub async fn handle_mpv_event(app: &App, mpv: &Mpv, event: &Event<'_>) -> Result<bool> { + match event { + Event::EndFile(r) => match r.reason { + EndFileReason::Eof => { + info!("Mpv reached the end of the current video. Marking it watched."); + mark_video_watched(app, mpv).await?; + reload_mpv_playlist(app, mpv, None, None).await?; + } + EndFileReason::Stop => { + // This reason is incredibly ambiguous. It _both_ means actually pausing a + // video and going to the next one in the playlist. + // Oh, and it's also called, when a video is removed from the playlist (at + // least via "playlist-remove current") + info!("Paused video (or went to next playlist entry); Doing nothing"); + } + EndFileReason::Quit => { + info!("Mpv quit. Exiting playback"); + + save_watch_progress(app, mpv).await?; + + return Ok(true); + } + EndFileReason::Error => { + unreachable!("This should have been raised as a separate error") + } + EndFileReason::Redirect => { + // TODO: We probably need to handle this somehow <2025-02-17> + } + }, + Event::StartFile(_) => { + let mpv_pos = usize::try_from(mpv.get_property::<i64>("playlist-pos")?) + .expect("The value is strictly positive"); + + let next_video = { + let yt_pos = get::current_playlist_index(app).await?.map(usize::from); + + if (Some(mpv_pos) != yt_pos) || yt_pos.is_none() { + let playlist = get::playlist(app).await?; + let video = playlist + .get(PlaylistIndex::from(mpv_pos)) + .expect("The mpv pos should not be out of bounds"); + + set::focused( + app, + &video.extractor_hash, + get::currently_focused_video(app) + .await? + .as_ref() + .map(|v| &v.extractor_hash), + ) + .await?; + + video.extractor_hash + } else { + get::currently_focused_video(app) + .await? + .expect("We have a focused video") + .extractor_hash + } + }; + + apply_video_options(app, mpv, &next_video).await?; + } + Event::Seek => { + save_watch_progress(app, mpv).await?; + } + Event::ClientMessage(a) => { + debug!("Got Client Message event: '{}'", a.join(" ")); + + match a.as_slice() { + &["yt-comments-external"] => { + client_messages::handle_yt_comments_external(app).await?; + } + &["yt-comments-local"] => { + client_messages::handle_yt_comments_local(app, mpv).await?; + } + + &["yt-description-external"] => { + client_messages::handle_yt_description_external(app).await?; + } + &["yt-description-local"] => { + client_messages::handle_yt_description_local(app, mpv).await?; + } + + &["yt-mark-picked"] => { + let current_video = get::currently_focused_video(app) + .await? + .expect("This should exist at this point"); + let current_index = get::current_playlist_index(app) + .await? + .expect("This should exist, as we can mark this video picked"); + + save_watch_progress(app, mpv).await?; + + set::video_status( + app, + ¤t_video.extractor_hash, + VideoStatus::Pick, + Some(current_video.priority), + ) + .await?; + + reload_mpv_playlist(app, mpv, None, Some(current_index)).await?; + mpv_message(mpv, "Marked the video as picked", Duration::from_secs(3))?; + } + &["yt-mark-watched"] => { + let current_index = get::current_playlist_index(app) + .await? + .expect("This should exist, as we can mark this video picked"); + mark_video_watched(app, mpv).await?; + + reload_mpv_playlist(app, mpv, None, Some(current_index)).await?; + mpv_message(mpv, "Marked the video watched", Duration::from_secs(3))?; + } + &["yt-check-new-videos"] => { + reload_mpv_playlist(app, mpv, None, None).await?; + } + other => { + debug!("Unknown message: {}", other.join(" ")); + } + } + } + _ => {} + } + + Ok(false) +} |