From 394d4f7d105dadd7b516f198b0d6a9dda2d3f1a5 Mon Sep 17 00:00:00 2001 From: Benedikt Peetz Date: Fri, 13 Jun 2025 21:18:16 +0200 Subject: refactor(yt): Move to `crates/yt` Having one crate outside the `crates` directory is just weird. --- crates/yt/Cargo.toml | 65 ++++ crates/yt/src/ansi_escape_codes.rs | 26 ++ crates/yt/src/app.rs | 50 +++ crates/yt/src/cache/mod.rs | 105 ++++++ crates/yt/src/cli.rs | 371 +++++++++++++++++++++ crates/yt/src/comments/comment.rs | 152 +++++++++ crates/yt/src/comments/description.rs | 46 +++ crates/yt/src/comments/display.rs | 118 +++++++ crates/yt/src/comments/mod.rs | 167 ++++++++++ crates/yt/src/comments/output.rs | 53 +++ crates/yt/src/config/default.rs | 110 ++++++ crates/yt/src/config/definitions.rs | 67 ++++ crates/yt/src/config/file_system.rs | 120 +++++++ crates/yt/src/config/mod.rs | 76 +++++ crates/yt/src/constants.rs | 12 + crates/yt/src/download/download_options.rs | 118 +++++++ crates/yt/src/download/mod.rs | 366 ++++++++++++++++++++ crates/yt/src/download/progress_hook.rs | 188 +++++++++++ crates/yt/src/main.rs | 247 ++++++++++++++ crates/yt/src/select/cmds/add.rs | 191 +++++++++++ crates/yt/src/select/cmds/mod.rs | 111 ++++++ crates/yt/src/select/mod.rs | 176 ++++++++++ crates/yt/src/select/selection_file/duration.rs | 185 ++++++++++ crates/yt/src/select/selection_file/help.str | 12 + .../yt/src/select/selection_file/help.str.license | 10 + crates/yt/src/select/selection_file/mod.rs | 32 ++ crates/yt/src/status/mod.rs | 129 +++++++ crates/yt/src/storage/migrate/mod.rs | 279 ++++++++++++++++ .../yt/src/storage/migrate/sql/0_Empty_to_Zero.sql | 72 ++++ .../yt/src/storage/migrate/sql/1_Zero_to_One.sql | 28 ++ crates/yt/src/storage/migrate/sql/2_One_to_Two.sql | 11 + .../yt/src/storage/migrate/sql/3_Two_to_Three.sql | 85 +++++ crates/yt/src/storage/mod.rs | 14 + crates/yt/src/storage/subscriptions.rs | 141 ++++++++ crates/yt/src/storage/video_database/downloader.rs | 130 ++++++++ .../src/storage/video_database/extractor_hash.rs | 163 +++++++++ crates/yt/src/storage/video_database/get/mod.rs | 307 +++++++++++++++++ .../video_database/get/playlist/iterator.rs | 101 ++++++ .../src/storage/video_database/get/playlist/mod.rs | 167 ++++++++++ crates/yt/src/storage/video_database/mod.rs | 329 ++++++++++++++++++ crates/yt/src/storage/video_database/notify.rs | 77 +++++ crates/yt/src/storage/video_database/set/mod.rs | 333 ++++++++++++++++++ .../yt/src/storage/video_database/set/playlist.rs | 101 ++++++ crates/yt/src/subscribe/mod.rs | 184 ++++++++++ crates/yt/src/unreachable.rs | 50 +++ crates/yt/src/update/mod.rs | 203 +++++++++++ crates/yt/src/update/updater.rs | 167 ++++++++++ crates/yt/src/version/mod.rs | 63 ++++ crates/yt/src/videos/display/format_video.rs | 94 ++++++ crates/yt/src/videos/display/mod.rs | 229 +++++++++++++ crates/yt/src/videos/mod.rs | 67 ++++ crates/yt/src/watch/mod.rs | 178 ++++++++++ crates/yt/src/watch/playlist.rs | 99 ++++++ .../watch/playlist_handler/client_messages/mod.rs | 98 ++++++ crates/yt/src/watch/playlist_handler/mod.rs | 342 +++++++++++++++++++ 55 files changed, 7415 insertions(+) create mode 100644 crates/yt/Cargo.toml create mode 100644 crates/yt/src/ansi_escape_codes.rs create mode 100644 crates/yt/src/app.rs create mode 100644 crates/yt/src/cache/mod.rs create mode 100644 crates/yt/src/cli.rs create mode 100644 crates/yt/src/comments/comment.rs create mode 100644 crates/yt/src/comments/description.rs create mode 100644 crates/yt/src/comments/display.rs create mode 100644 crates/yt/src/comments/mod.rs create mode 100644 crates/yt/src/comments/output.rs create mode 100644 crates/yt/src/config/default.rs create mode 100644 crates/yt/src/config/definitions.rs create mode 100644 crates/yt/src/config/file_system.rs create mode 100644 crates/yt/src/config/mod.rs create mode 100644 crates/yt/src/constants.rs create mode 100644 crates/yt/src/download/download_options.rs create mode 100644 crates/yt/src/download/mod.rs create mode 100644 crates/yt/src/download/progress_hook.rs create mode 100644 crates/yt/src/main.rs create mode 100644 crates/yt/src/select/cmds/add.rs create mode 100644 crates/yt/src/select/cmds/mod.rs create mode 100644 crates/yt/src/select/mod.rs create mode 100644 crates/yt/src/select/selection_file/duration.rs create mode 100644 crates/yt/src/select/selection_file/help.str create mode 100644 crates/yt/src/select/selection_file/help.str.license create mode 100644 crates/yt/src/select/selection_file/mod.rs create mode 100644 crates/yt/src/status/mod.rs create mode 100644 crates/yt/src/storage/migrate/mod.rs create mode 100644 crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql create mode 100644 crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql create mode 100644 crates/yt/src/storage/migrate/sql/2_One_to_Two.sql create mode 100644 crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql create mode 100644 crates/yt/src/storage/mod.rs create mode 100644 crates/yt/src/storage/subscriptions.rs create mode 100644 crates/yt/src/storage/video_database/downloader.rs create mode 100644 crates/yt/src/storage/video_database/extractor_hash.rs create mode 100644 crates/yt/src/storage/video_database/get/mod.rs create mode 100644 crates/yt/src/storage/video_database/get/playlist/iterator.rs create mode 100644 crates/yt/src/storage/video_database/get/playlist/mod.rs create mode 100644 crates/yt/src/storage/video_database/mod.rs create mode 100644 crates/yt/src/storage/video_database/notify.rs create mode 100644 crates/yt/src/storage/video_database/set/mod.rs create mode 100644 crates/yt/src/storage/video_database/set/playlist.rs create mode 100644 crates/yt/src/subscribe/mod.rs create mode 100644 crates/yt/src/unreachable.rs create mode 100644 crates/yt/src/update/mod.rs create mode 100644 crates/yt/src/update/updater.rs create mode 100644 crates/yt/src/version/mod.rs create mode 100644 crates/yt/src/videos/display/format_video.rs create mode 100644 crates/yt/src/videos/display/mod.rs create mode 100644 crates/yt/src/videos/mod.rs create mode 100644 crates/yt/src/watch/mod.rs create mode 100644 crates/yt/src/watch/playlist.rs create mode 100644 crates/yt/src/watch/playlist_handler/client_messages/mod.rs create mode 100644 crates/yt/src/watch/playlist_handler/mod.rs (limited to 'crates') diff --git a/crates/yt/Cargo.toml b/crates/yt/Cargo.toml new file mode 100644 index 0000000..17d4016 --- /dev/null +++ b/crates/yt/Cargo.toml @@ -0,0 +1,65 @@ +# yt - A fully featured command line YouTube client +# +# Copyright (C) 2024 Benedikt Peetz +# Copyright (C) 2025 Benedikt Peetz +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Yt. +# +# You should have received a copy of the License along with this program. +# If not, see . + +[package] +name = "yt" +description = "A fully featured command line YouTube client" +keywords = [] +categories = [] +default-run = "yt" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +anyhow = "1.0.98" +blake3 = "1.8.2" +chrono = { version = "0.4.41", features = ["now"] } +chrono-humanize = "0.2.3" +clap = { version = "4.5.40", features = ["derive"] } +futures = "0.3.31" +nucleo-matcher = "0.3.1" +owo-colors = "4.2.1" +regex = "1.11.1" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "sqlite"] } +stderrlog = "0.6.0" +tempfile = "3.20.0" +toml = "0.8.23" +trinitry = { version = "0.2.2" } +xdg = "3.0.0" +bytes.workspace = true +libmpv2.workspace = true +log.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +url.workspace = true +yt_dlp.workspace = true +termsize.workspace = true +uu_fmt.workspace = true +notify = { version = "8.0.0", default-features = false } + +[[bin]] +name = "yt" +doc = false +path = "src/main.rs" + +[dev-dependencies] + +[lints] +workspace = true + +[package.metadata.docs.rs] +all-features = true diff --git a/crates/yt/src/ansi_escape_codes.rs b/crates/yt/src/ansi_escape_codes.rs new file mode 100644 index 0000000..ae1805d --- /dev/null +++ b/crates/yt/src/ansi_escape_codes.rs @@ -0,0 +1,26 @@ +// see: https://en.wikipedia.org/wiki/ANSI_escape_code#Control_Sequence_Introducer_commands +const CSI: &str = "\x1b["; +pub fn erase_in_display_from_cursor() { + print!("{CSI}0J"); +} +pub fn cursor_up(number: usize) { + // HACK(@bpeetz): The default is `1` and running this command with a + // number of `0` results in it using the default (i.e., `1`) <2025-03-25> + if number != 0 { + print!("{CSI}{number}A"); + } +} + +pub fn clear_whole_line() { + eprint!("{CSI}2K"); +} +pub fn move_to_col(x: usize) { + eprint!("{CSI}{x}G"); +} + +pub fn hide_cursor() { + eprint!("{CSI}?25l"); +} +pub fn show_cursor() { + eprint!("{CSI}?25h"); +} diff --git a/crates/yt/src/app.rs b/crates/yt/src/app.rs new file mode 100644 index 0000000..15a9388 --- /dev/null +++ b/crates/yt/src/app.rs @@ -0,0 +1,50 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use anyhow::{Context, Result}; +use log::warn; +use sqlx::{SqlitePool, sqlite::SqliteConnectOptions}; + +use crate::{config::Config, storage::migrate::migrate_db}; + +#[derive(Debug)] +pub struct App { + pub database: SqlitePool, + pub config: Config, +} + +impl App { + pub async fn new(config: Config, should_migrate_db: bool) -> Result { + let options = SqliteConnectOptions::new() + .filename(&config.paths.database_path) + .optimize_on_close(true, None) + .create_if_missing(true); + + let pool = SqlitePool::connect_with(options) + .await + .context("Failed to connect to database!")?; + + let app = App { + database: pool, + config, + }; + + if should_migrate_db { + migrate_db(&app) + .await + .context("Failed to migrate db to new version")?; + } else { + warn!("Skipping database migration."); + } + + Ok(app) + } +} diff --git a/crates/yt/src/cache/mod.rs b/crates/yt/src/cache/mod.rs new file mode 100644 index 0000000..83d5ee0 --- /dev/null +++ b/crates/yt/src/cache/mod.rs @@ -0,0 +1,105 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use anyhow::{Context, Result}; +use log::{debug, info}; +use tokio::fs; + +use crate::{ + app::App, + storage::video_database::{ + Video, VideoStatus, VideoStatusMarker, downloader::set_video_cache_path, get, + }, +}; + +async fn invalidate_video(app: &App, video: &Video, hard: bool) -> Result<()> { + info!("Invalidating cache of video: '{}'", video.title); + + if hard { + if let VideoStatus::Cached { + cache_path: path, .. + } = &video.status + { + info!("Removing cached video at: '{}'", path.display()); + if let Err(err) = fs::remove_file(path).await.map_err(|err| err.kind()) { + match err { + std::io::ErrorKind::NotFound => { + // The path is already gone + debug!( + "Not actually removing path: '{}'. It is already gone.", + path.display() + ); + } + err => Err(std::io::Error::from(err)).with_context(|| { + format!( + "Failed to delete video ('{}') cache path: '{}'.", + video.title, + path.display() + ) + })?, + } + } + } + } + + set_video_cache_path(app, &video.extractor_hash, None).await?; + + Ok(()) +} + +pub async fn invalidate(app: &App, hard: bool) -> Result<()> { + let all_cached_things = get::videos(app, &[VideoStatusMarker::Cached]).await?; + + info!("Got videos to invalidate: '{}'", all_cached_things.len()); + + for video in all_cached_things { + invalidate_video(app, &video, hard).await?; + } + + Ok(()) +} + +/// # Panics +/// Only if internal assertions fail. +pub async fn maintain(app: &App, all: bool) -> Result<()> { + let domain = if all { + VideoStatusMarker::ALL.as_slice() + } else { + &[VideoStatusMarker::Watch, VideoStatusMarker::Cached] + }; + + let cached_videos = get::videos(app, domain).await?; + + let mut found_focused = 0; + for vid in cached_videos { + if let VideoStatus::Cached { + cache_path: path, + is_focused, + } = &vid.status + { + info!("Checking if path ('{}') exists", path.display()); + if !path.exists() { + invalidate_video(app, &vid, false).await?; + } + + if *is_focused { + found_focused += 1; + } + } + } + + assert!( + found_focused <= 1, + "Only one video can be focused at a time" + ); + + Ok(()) +} diff --git a/crates/yt/src/cli.rs b/crates/yt/src/cli.rs new file mode 100644 index 0000000..de7a5b8 --- /dev/null +++ b/crates/yt/src/cli.rs @@ -0,0 +1,371 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::{path::PathBuf, str::FromStr}; + +use anyhow::Context; +use bytes::Bytes; +use chrono::NaiveDate; +use clap::{ArgAction, Args, Parser, Subcommand}; +use url::Url; + +use crate::{ + select::selection_file::duration::MaybeDuration, + storage::video_database::extractor_hash::LazyExtractorHash, +}; + +#[derive(Parser, Debug)] +#[clap(author, about, long_about = None)] +#[allow(clippy::module_name_repetitions)] +/// An command line interface to select, download and watch videos +pub struct CliArgs { + #[command(subcommand)] + /// The subcommand to execute [default: select] + pub command: Option, + + /// Show the version and exit + #[arg(long, short = 'V', action= ArgAction::SetTrue)] + pub version: bool, + + /// Do not perform database migration before starting. + /// Setting this could cause runtime database access errors. + #[arg(long, short, action=ArgAction::SetTrue, default_value_t = false)] + pub no_migrate_db: bool, + + /// Display colors [defaults to true, if the config file has no value] + #[arg(long, short = 'C')] + pub color: Option, + + /// Set the path to the videos.db. This overrides the default and the config file. + #[arg(long, short)] + pub db_path: Option, + + /// Set the path to the config.toml. + /// This overrides the default. + #[arg(long, short)] + pub config_path: Option, + + /// Increase message verbosity + #[arg(long="verbose", short = 'v', action = ArgAction::Count)] + pub verbosity: u8, + + /// Silence all output + #[arg(long, short = 'q')] + pub quiet: bool, +} + +#[derive(Subcommand, Debug)] +pub enum Command { + /// Download and cache URLs + Download { + /// Forcefully re-download all cached videos (i.e. delete the cache path, then download). + #[arg(short, long)] + force: bool, + + /// The maximum size the download dir should have. Beware that the value must be given in + /// bytes. + #[arg(short, long, value_parser = byte_parser)] + max_cache_size: Option, + }, + + /// Select, download and watch in one command. + Sedowa {}, + /// Download and watch in one command. + Dowa {}, + + /// Work with single videos + Videos { + #[command(subcommand)] + cmd: VideosCommand, + }, + + /// Watch the already cached (and selected) videos + Watch {}, + + /// Visualize the current playlist + Playlist { + /// Linger and display changes + #[arg(short, long)] + watch: bool, + }, + + /// Show, which videos have been selected to be watched (and their cache status) + Status {}, + + /// Show, the configuration options in effect + Config {}, + + /// Display the comments of the currently playing video + Comments {}, + /// Display the description of the currently playing video + Description {}, + + /// Manipulate the video cache in the database + #[command(visible_alias = "db")] + Database { + #[command(subcommand)] + command: CacheCommand, + }, + + /// Change the state of videos in the database (the default) + Select { + #[command(subcommand)] + cmd: Option, + }, + + /// Update the video database + Update { + #[arg(short, long)] + /// The number of videos to updating + max_backlog: Option, + + #[arg(short, long)] + /// The subscriptions to update (can be given multiple times) + subscriptions: Vec, + }, + + /// Manipulate subscription + #[command(visible_alias = "subs")] + Subscriptions { + #[command(subcommand)] + cmd: SubscriptionCommand, + }, +} + +fn byte_parser(input: &str) -> Result { + Ok(input + .parse::() + .with_context(|| format!("Failed to parse '{input}' as bytes!"))? + .as_u64()) +} + +impl Default for Command { + fn default() -> Self { + Self::Select { + cmd: Some(SelectCommand::default()), + } + } +} + +#[derive(Subcommand, Clone, Debug)] +pub enum VideosCommand { + /// List the videos in the database + #[command(visible_alias = "ls")] + List { + /// An optional search query to limit the results + #[arg(action = ArgAction::Append)] + search_query: Option, + + /// The number of videos to show + #[arg(short, long)] + limit: Option, + }, + + /// Get detailed information about a video + Info { + /// The short hash of the video + hash: LazyExtractorHash, + }, +} + +#[derive(Subcommand, Clone, Debug)] +pub enum SubscriptionCommand { + /// Subscribe to an URL + Add { + #[arg(short, long)] + /// The human readable name of the subscription + name: Option, + + /// The URL to listen to + url: Url, + }, + + /// Unsubscribe from an URL + Remove { + /// The human readable name of the subscription + name: String, + }, + + /// Import a bunch of URLs as subscriptions. + Import { + /// The file containing the URLs. Will use Stdin otherwise. + file: Option, + + /// Remove any previous subscriptions + #[arg(short, long)] + force: bool, + }, + /// Write all subscriptions in an format understood by `import` + Export {}, + + /// List all subscriptions + List {}, +} + +#[derive(Clone, Debug, Args)] +#[command(infer_subcommands = true)] +/// Mark the video given by the hash to be watched +pub struct SharedSelectionCommandArgs { + /// The ordering priority (higher means more at the top) + #[arg(short, long)] + pub priority: Option, + + /// The subtitles to download (e.g. 'en,de,sv') + #[arg(short = 'l', long)] + pub subtitle_langs: Option, + + /// The speed to set mpv to + #[arg(short, long)] + pub speed: Option, + + /// The short extractor hash + pub hash: LazyExtractorHash, + + pub title: Option, + + pub date: Option, + + pub publisher: Option, + + pub duration: Option, + + pub url: Option, +} +#[derive(Clone, Debug, Copy)] +pub struct OptionalNaiveDate { + pub date: Option, +} +impl FromStr for OptionalNaiveDate { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result { + if v == "[No release date]" { + Ok(Self { date: None }) + } else { + Ok(Self { + date: Some(NaiveDate::from_str(v)?), + }) + } + } +} +#[derive(Clone, Debug)] +pub struct OptionalPublisher { + pub publisher: Option, +} +impl FromStr for OptionalPublisher { + type Err = anyhow::Error; + fn from_str(v: &str) -> Result { + if v == "[No author]" { + Ok(Self { publisher: None }) + } else { + Ok(Self { + publisher: Some(v.to_owned()), + }) + } + } +} + +#[derive(Subcommand, Clone, Debug)] +// NOTE: Keep this in sync with the [`constants::HELP_STR`] constant. <2024-08-20> +// NOTE: Also keep this in sync with the `tree-sitter-yts/grammar.js`. <2024-11-04> +pub enum SelectCommand { + /// Open a `git rebase` like file to select the videos to watch (the default) + File { + /// Include done (watched, dropped) videos + #[arg(long, short)] + done: bool, + + /// Use the last selection file (useful if you've spend time on it and want to get it again) + #[arg(long, short, conflicts_with = "done")] + use_last_selection: bool, + }, + + /// Add a video to the database + /// + /// This optionally supports to add a playlist. + /// When a playlist is added, the `start` and `stop` arguments can be used to select which + /// playlist entries to include. + #[command(visible_alias = "a")] + Add { + urls: Vec, + + /// Start adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 's', long)] + start: Option, + + /// Stop adding playlist entries at this playlist index (zero based and inclusive) + #[arg(short = 'e', long)] + stop: Option, + }, + + /// Mark the video given by the hash to be watched + #[command(visible_alias = "w")] + Watch { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash to be dropped + #[command(visible_alias = "d")] + Drop { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Mark the video given by the hash as already watched + #[command(visible_alias = "wd")] + Watched { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Open the video URL in Firefox's `timesinks.youtube` profile + #[command(visible_alias = "u")] + Url { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, + + /// Reset the videos status to 'Pick' + #[command(visible_alias = "p")] + Pick { + #[command(flatten)] + shared: SharedSelectionCommandArgs, + }, +} +impl Default for SelectCommand { + fn default() -> Self { + Self::File { + done: false, + use_last_selection: false, + } + } +} + +#[derive(Subcommand, Clone, Copy, Debug)] +pub enum CacheCommand { + /// Invalidate all cache entries + Invalidate { + /// Also delete the cache path + #[arg(short, long)] + hard: bool, + }, + + /// Perform basic maintenance operations on the database. + /// This helps recovering from invalid db states after a crash (or force exit via ). + /// + /// 1. Check every path for validity (removing all invalid cache entries) + #[command(verbatim_doc_comment)] + Maintain { + /// Check every video (otherwise only the videos to be watched are checked) + #[arg(short, long)] + all: bool, + }, +} diff --git a/crates/yt/src/comments/comment.rs b/crates/yt/src/comments/comment.rs new file mode 100644 index 0000000..5bc939c --- /dev/null +++ b/crates/yt/src/comments/comment.rs @@ -0,0 +1,152 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use serde::{Deserialize, Deserializer, Serialize}; +use url::Url; + +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub enum Parent { + Root, + Id(String), +} + +impl Parent { + #[must_use] + pub fn id(&self) -> Option<&str> { + if let Self::Id(id) = self { + Some(id) + } else { + None + } + } +} + +impl From for Parent { + fn from(value: String) -> Self { + if value == "root" { + Self::Root + } else { + Self::Id(value) + } + } +} + +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[serde(from = "String")] +#[serde(deny_unknown_fields)] +pub struct Id { + pub id: String, +} +impl From for Id { + fn from(value: String) -> Self { + Self { + // Take the last element if the string is split with dots, otherwise take the full id + id: value.split('.').last().unwrap_or(&value).to_owned(), + } + } +} + +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq, PartialOrd, Ord)] +#[allow(clippy::struct_excessive_bools)] +pub struct Comment { + pub id: Id, + pub text: String, + #[serde(default = "zero")] + pub like_count: u32, + pub is_pinned: bool, + pub author_id: String, + #[serde(default = "unknown")] + pub author: String, + pub author_is_verified: bool, + pub author_thumbnail: Url, + pub parent: Parent, + #[serde(deserialize_with = "edited_from_time_text", alias = "_time_text")] + pub edited: bool, + // Can't also be deserialized, as it's already used in 'edited' + // _time_text: String, + pub timestamp: i64, + pub author_url: Option, + pub author_is_uploader: bool, + pub is_favorited: bool, +} + +fn unknown() -> String { + "".to_string() +} +fn zero() -> u32 { + 0 +} +fn edited_from_time_text<'de, D>(d: D) -> Result +where + D: Deserializer<'de>, +{ + let s = String::deserialize(d)?; + if s.contains(" (edited)") { + Ok(true) + } else { + Ok(false) + } +} + +#[derive(Debug, Clone)] +#[allow(clippy::module_name_repetitions)] +pub struct CommentExt { + pub value: Comment, + pub replies: Vec, +} + +#[derive(Debug, Default)] +pub struct Comments { + pub(super) vec: Vec, +} + +impl Comments { + pub fn new() -> Self { + Self::default() + } + pub fn push(&mut self, value: CommentExt) { + self.vec.push(value); + } + pub fn get_mut(&mut self, key: &str) -> Option<&mut CommentExt> { + self.vec.iter_mut().filter(|c| c.value.id.id == key).last() + } + pub fn insert(&mut self, key: &str, value: CommentExt) { + let parent = self + .vec + .iter_mut() + .filter(|c| c.value.id.id == key) + .last() + .expect("One of these should exist"); + parent.push_reply(value); + } +} +impl CommentExt { + pub fn push_reply(&mut self, value: CommentExt) { + self.replies.push(value); + } + pub fn get_mut_reply(&mut self, key: &str) -> Option<&mut CommentExt> { + self.replies + .iter_mut() + .filter(|c| c.value.id.id == key) + .last() + } +} + +impl From for CommentExt { + fn from(value: Comment) -> Self { + Self { + replies: vec![], + value, + } + } +} diff --git a/crates/yt/src/comments/description.rs b/crates/yt/src/comments/description.rs new file mode 100644 index 0000000..e8cb29d --- /dev/null +++ b/crates/yt/src/comments/description.rs @@ -0,0 +1,46 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use crate::{ + App, + comments::output::display_fmt_and_less, + storage::video_database::{Video, get}, + unreachable::Unreachable, +}; + +use anyhow::{Result, bail}; +use yt_dlp::{InfoJson, json_cast}; + +pub async fn description(app: &App) -> Result<()> { + let description = get(app).await?; + display_fmt_and_less(description).await?; + + Ok(()) +} + +pub async fn get(app: &App) -> Result { + let currently_playing_video: Video = + if let Some(video) = get::currently_focused_video(app).await? { + video + } else { + bail!("Could not find a currently playing video!"); + }; + + let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( + "A currently *playing* must be cached. And thus the info.json should be available", + ); + + Ok(info_json + .get("description") + .map(|val| json_cast!(val, as_str)) + .unwrap_or("") + .to_owned()) +} diff --git a/crates/yt/src/comments/display.rs b/crates/yt/src/comments/display.rs new file mode 100644 index 0000000..6166b2b --- /dev/null +++ b/crates/yt/src/comments/display.rs @@ -0,0 +1,118 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::fmt::Write; + +use chrono::{Local, TimeZone}; +use chrono_humanize::{Accuracy, HumanTime, Tense}; + +use crate::comments::comment::CommentExt; + +use super::comment::Comments; + +impl Comments { + pub fn render(&self, color: bool) -> String { + self.render_help(color).expect("This should never fail.") + } + + fn render_help(&self, color: bool) -> Result { + macro_rules! c { + ($color_str:expr, $write:ident, $color:expr) => { + if $color { + $write.write_str(concat!("\x1b[", $color_str, "m"))? + } + }; + } + + fn format( + comment: &CommentExt, + f: &mut String, + ident_count: u32, + color: bool, + ) -> std::fmt::Result { + let ident = &(0..ident_count).map(|_| " ").collect::(); + let value = &comment.value; + + f.write_str(ident)?; + + if value.author_is_uploader { + c!("91;1", f, color); + } else { + c!("35", f, color); + } + + f.write_str(&value.author)?; + c!("0", f, color); + if value.edited || value.is_favorited { + f.write_str("[")?; + if value.edited { + f.write_str("")?; + } + if value.edited && value.is_favorited { + f.write_str(" ")?; + } + if value.is_favorited { + f.write_str("")?; + } + f.write_str("]")?; + } + + c!("36;1", f, color); + write!( + f, + " {}", + HumanTime::from( + Local + .timestamp_opt(value.timestamp, 0) + .single() + .expect("This should be valid") + ) + .to_text_en(Accuracy::Rough, Tense::Past) + )?; + c!("0", f, color); + + // c!("31;1", f); + // f.write_fmt(format_args!(" [{}]", comment.value.like_count))?; + // c!("0", f); + + f.write_str(":\n")?; + f.write_str(ident)?; + + f.write_str(&value.text.replace('\n', &format!("\n{ident}")))?; + f.write_str("\n")?; + + if comment.replies.is_empty() { + f.write_str("\n")?; + } else { + let mut children = comment.replies.clone(); + children.sort_by(|a, b| a.value.timestamp.cmp(&b.value.timestamp)); + + for child in children { + format(&child, f, ident_count + 4, color)?; + } + } + + Ok(()) + } + + let mut f = String::new(); + + if !&self.vec.is_empty() { + let mut children = self.vec.clone(); + children.sort_by(|a, b| b.value.like_count.cmp(&a.value.like_count)); + + for child in children { + format(&child, &mut f, 0, color)?; + } + } + Ok(f) + } +} diff --git a/crates/yt/src/comments/mod.rs b/crates/yt/src/comments/mod.rs new file mode 100644 index 0000000..876146d --- /dev/null +++ b/crates/yt/src/comments/mod.rs @@ -0,0 +1,167 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::mem; + +use anyhow::{Result, bail}; +use comment::{Comment, CommentExt, Comments, Parent}; +use output::display_fmt_and_less; +use regex::Regex; +use yt_dlp::{InfoJson, json_cast}; + +use crate::{ + app::App, + storage::video_database::{Video, get}, + unreachable::Unreachable, +}; + +mod comment; +mod display; +pub mod output; + +pub mod description; +pub use description::*; + +#[allow(clippy::too_many_lines)] +pub async fn get(app: &App) -> Result { + let currently_playing_video: Video = + if let Some(video) = get::currently_focused_video(app).await? { + video + } else { + bail!("Could not find a currently playing video!"); + }; + + let info_json: InfoJson = get::video_info_json(¤tly_playing_video)?.unreachable( + "A currently *playing* video must be cached. And thus the info.json should be available", + ); + + let base_comments = if let Some(comments) = info_json.get("comments") { + json_cast!(comments, as_array) + } else { + bail!( + "The video ('{}') does not have comments!", + info_json + .get("title") + .map(|val| json_cast!(val, as_str)) + .unwrap_or("") + ) + }; + + let mut comments = Comments::new(); + for c in base_comments { + let c: Comment = serde_json::from_value(c.to_owned())?; + if let Parent::Id(id) = &c.parent { + comments.insert(&(id.clone()), CommentExt::from(c)); + } else { + comments.push(CommentExt::from(c)); + } + } + + comments.vec.iter_mut().for_each(|comment| { + let replies = mem::take(&mut comment.replies); + let mut output_replies: Vec = vec![]; + + let re = Regex::new(r"\u{200b}?(@[^\t\s]+)\u{200b}?").unreachable("This is hardcoded"); + for reply in replies { + if let Some(replyee_match) = re.captures(&reply.value.text){ + let full_match = replyee_match.get(0).unreachable("This will always exist"); + let text = reply. + value. + text[0..full_match.start()] + .to_owned() + + + &reply + .value + .text[full_match.end()..]; + let text: &str = text.trim().trim_matches('\u{200b}'); + + let replyee = replyee_match.get(1).unreachable("This should also exist").as_str(); + + + if let Some(parent) = output_replies + .iter_mut() + // .rev() + .flat_map(|com| &mut com.replies) + .flat_map(|com| &mut com.replies) + .flat_map(|com| &mut com.replies) + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else if let Some(parent) = output_replies + .iter_mut() + // .rev() + .flat_map(|com| &mut com.replies) + .flat_map(|com| &mut com.replies) + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else if let Some(parent) = output_replies + .iter_mut() + // .rev() + .flat_map(|com| &mut com.replies) + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else if let Some(parent) = output_replies.iter_mut() + // .rev() + .filter(|com| com.value.author == replyee) + .last() + { + parent.replies.push(CommentExt::from(Comment { + text: text.to_owned(), + ..reply.value + })); + } else { + eprintln!( + "Failed to find a parent for ('{}') both directly and via replies! The reply text was:\n'{}'\n", + replyee, + reply.value.text + ); + output_replies.push(reply); + } + } else { + output_replies.push(reply); + } + } + comment.replies = output_replies; + }); + + Ok(comments) +} + +pub async fn comments(app: &App) -> Result<()> { + let comments = get(app).await?; + + display_fmt_and_less(comments.render(true)).await?; + + Ok(()) +} + +#[cfg(test)] +mod test { + #[test] + fn test_string_replacement() { + let s = "A \n\nB\n\nC".to_owned(); + assert_eq!("A \n \n B\n \n C", s.replace('\n', "\n ")); + } +} diff --git a/crates/yt/src/comments/output.rs b/crates/yt/src/comments/output.rs new file mode 100644 index 0000000..cb3a9c4 --- /dev/null +++ b/crates/yt/src/comments/output.rs @@ -0,0 +1,53 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::{ + io::Write, + process::{Command, Stdio}, +}; + +use anyhow::{Context, Result}; +use uu_fmt::{FmtOptions, process_text}; + +use crate::unreachable::Unreachable; + +pub async fn display_fmt_and_less(input: String) -> Result<()> { + let mut less = Command::new("less") + .args(["--raw-control-chars"]) + .stdin(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .context("Failed to run less")?; + + let input = format_text(&input); + let mut stdin = less.stdin.take().context("Failed to open stdin")?; + std::thread::spawn(move || { + stdin + .write_all(input.as_bytes()) + .unreachable("Should be able to write to the stdin of less"); + }); + + let _ = less.wait().context("Failed to await less")?; + + Ok(()) +} + +#[must_use] +pub fn format_text(input: &str) -> String { + let width = termsize::get().map_or(90, |size| size.cols); + let fmt_opts = FmtOptions { + uniform: true, + split_only: true, + ..FmtOptions::new(Some(width as usize), None, Some(4)) + }; + + process_text(input, &fmt_opts) +} diff --git a/crates/yt/src/config/default.rs b/crates/yt/src/config/default.rs new file mode 100644 index 0000000..4ed643b --- /dev/null +++ b/crates/yt/src/config/default.rs @@ -0,0 +1,110 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::path::PathBuf; + +use anyhow::{Context, Result}; + +fn get_runtime_path(name: &'static str) -> Result { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_runtime_file(name) + .with_context(|| format!("Failed to place runtime file: '{name}'")) +} +fn get_data_path(name: &'static str) -> Result { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_data_file(name) + .with_context(|| format!("Failed to place data file: '{name}'")) +} +fn get_config_path(name: &'static str) -> Result { + let xdg_dirs = xdg::BaseDirectories::with_prefix(PREFIX); + xdg_dirs + .place_config_file(name) + .with_context(|| format!("Failed to place config file: '{name}'")) +} + +pub(super) fn create_path(path: PathBuf) -> Result { + if !path.exists() { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create the '{}' directory", path.display()))?; + } + } + + Ok(path) +} + +pub(crate) const PREFIX: &str = "yt"; + +pub(crate) mod global { + pub(crate) fn display_colors() -> bool { + // TODO: This should probably check if the output is a tty and otherwise return `false` <2025-02-14> + true + } +} + +pub(crate) mod select { + pub(crate) fn playback_speed() -> f64 { + 2.7 + } + pub(crate) fn subtitle_langs() -> &'static str { + "" + } +} + +pub(crate) mod watch { + pub(crate) fn local_displays_length() -> usize { + 1000 + } +} + +pub(crate) mod update { + pub(crate) fn max_backlog() -> usize { + 20 + } +} + +pub(crate) mod paths { + use std::{env::temp_dir, path::PathBuf}; + + use anyhow::Result; + + use super::{PREFIX, create_path, get_config_path, get_data_path, get_runtime_path}; + + // We download to the temp dir to avoid taxing the disk + pub(crate) fn download_dir() -> Result { + let temp_dir = temp_dir(); + + create_path(temp_dir.join(PREFIX)) + } + pub(crate) fn mpv_config_path() -> Result { + get_config_path("mpv.conf") + } + pub(crate) fn mpv_input_path() -> Result { + get_config_path("mpv.input.conf") + } + pub(crate) fn database_path() -> Result { + get_data_path("videos.sqlite") + } + pub(crate) fn config_path() -> Result { + get_config_path("config.toml") + } + pub(crate) fn last_selection_path() -> Result { + get_runtime_path("selected.yts") + } +} + +pub(crate) mod download { + pub(crate) fn max_cache_size() -> &'static str { + "3 GiB" + } +} diff --git a/crates/yt/src/config/definitions.rs b/crates/yt/src/config/definitions.rs new file mode 100644 index 0000000..ce8c0d4 --- /dev/null +++ b/crates/yt/src/config/definitions.rs @@ -0,0 +1,67 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::path::PathBuf; + +use serde::Deserialize; + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ConfigFile { + pub global: Option, + pub select: Option, + pub watch: Option, + pub paths: Option, + pub download: Option, + pub update: Option, +} + +#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] +#[serde(deny_unknown_fields)] +pub(crate) struct GlobalConfig { + pub display_colors: Option, +} + +#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] +#[serde(deny_unknown_fields)] +pub(crate) struct UpdateConfig { + pub max_backlog: Option, +} + +#[derive(Debug, Deserialize, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub(crate) struct DownloadConfig { + /// This will then be converted to an u64 + pub max_cache_size: Option, +} + +#[derive(Debug, Deserialize, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub(crate) struct SelectConfig { + pub playback_speed: Option, + pub subtitle_langs: Option, +} + +#[derive(Debug, Deserialize, PartialEq, Clone, Copy)] +#[serde(deny_unknown_fields)] +pub(crate) struct WatchConfig { + pub local_displays_length: Option, +} + +#[derive(Debug, Deserialize, PartialEq, Clone)] +#[serde(deny_unknown_fields)] +pub(crate) struct PathsConfig { + pub download_dir: Option, + pub mpv_config_path: Option, + pub mpv_input_path: Option, + pub database_path: Option, + pub last_selection_path: Option, +} diff --git a/crates/yt/src/config/file_system.rs b/crates/yt/src/config/file_system.rs new file mode 100644 index 0000000..2463e9d --- /dev/null +++ b/crates/yt/src/config/file_system.rs @@ -0,0 +1,120 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use crate::config::{DownloadConfig, PathsConfig, SelectConfig, WatchConfig}; + +use super::{ + Config, GlobalConfig, UpdateConfig, + default::{create_path, download, global, paths, select, update, watch}, +}; + +use std::{fs::read_to_string, path::PathBuf}; + +use anyhow::{Context, Result}; +use bytes::Bytes; + +macro_rules! get { + ($default:path, $config:expr, $key_one:ident, $($keys:ident),*) => { + { + let maybe_value = get!{@option $config, $key_one, $($keys),*}; + if let Some(value) = maybe_value { + value + } else { + $default().to_owned() + } + } + }; + + (@option $config:expr, $key_one:ident, $($keys:ident),*) => { + if let Some(key) = $config.$key_one.clone() { + get!{@option key, $($keys),*} + } else { + None + } + }; + (@option $config:expr, $key_one:ident) => { + $config.$key_one + }; + + (@path_if_none $config:expr, $option_default:expr, $default:path, $key_one:ident, $($keys:ident),*) => { + { + let maybe_download_dir: Option = + get! {@option $config, $key_one, $($keys),*}; + + let down_dir = if let Some(dir) = maybe_download_dir { + PathBuf::from(dir) + } else { + if let Some(path) = $option_default { + path + } else { + $default() + .with_context(|| format!("Failed to get default path for: '{}.{}'", stringify!($key_one), stringify!($($keys),*)))? + } + }; + create_path(down_dir)? + } + }; + (@path $config:expr, $default:path, $key_one:ident, $($keys:ident),*) => { + get! {@path_if_none $config, None, $default, $key_one, $($keys),*} + }; +} + +impl Config { + pub fn from_config_file( + db_path: Option, + config_path: Option, + display_colors: Option, + ) -> Result { + let config_file_path = + config_path.map_or_else(|| -> Result<_> { paths::config_path() }, Ok)?; + + let config: super::definitions::ConfigFile = + toml::from_str(&read_to_string(config_file_path).unwrap_or(String::new())) + .context("Failed to parse the config file as toml")?; + + Ok(Self { + global: GlobalConfig { + display_colors: { + let config_value: Option = get! {@option config, global, display_colors}; + + display_colors.unwrap_or(config_value.unwrap_or_else(global::display_colors)) + }, + }, + select: SelectConfig { + playback_speed: get! {select::playback_speed, config, select, playback_speed}, + subtitle_langs: get! {select::subtitle_langs, config, select, subtitle_langs}, + }, + watch: WatchConfig { + local_displays_length: get! {watch::local_displays_length, config, watch, local_displays_length}, + }, + update: UpdateConfig { + max_backlog: get! {update::max_backlog, config, update, max_backlog}, + }, + paths: PathsConfig { + download_dir: get! {@path config, paths::download_dir, paths, download_dir}, + mpv_config_path: get! {@path config, paths::mpv_config_path, paths, mpv_config_path}, + mpv_input_path: get! {@path config, paths::mpv_input_path, paths, mpv_input_path}, + database_path: get! {@path_if_none config, db_path, paths::database_path, paths, database_path}, + last_selection_path: get! {@path config, paths::last_selection_path, paths, last_selection_path}, + }, + download: DownloadConfig { + max_cache_size: { + let bytes_str: String = + get! {download::max_cache_size, config, download, max_cache_size}; + let number: Bytes = bytes_str + .parse() + .context("Failed to parse max_cache_size")?; + number + }, + }, + }) + } +} diff --git a/crates/yt/src/config/mod.rs b/crates/yt/src/config/mod.rs new file mode 100644 index 0000000..a10f7c2 --- /dev/null +++ b/crates/yt/src/config/mod.rs @@ -0,0 +1,76 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +#![allow(clippy::module_name_repetitions)] + +use std::path::PathBuf; + +use bytes::Bytes; +use serde::Serialize; + +mod default; +mod definitions; +pub mod file_system; + +#[derive(Serialize, Debug)] +pub struct Config { + pub global: GlobalConfig, + pub select: SelectConfig, + pub watch: WatchConfig, + pub paths: PathsConfig, + pub download: DownloadConfig, + pub update: UpdateConfig, +} +// These structures could get non-copy fields in the future. + +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct GlobalConfig { + pub display_colors: bool, +} +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct UpdateConfig { + pub max_backlog: usize, +} +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct DownloadConfig { + pub max_cache_size: Bytes, +} +#[derive(Serialize, Debug)] +pub struct SelectConfig { + pub playback_speed: f64, + pub subtitle_langs: String, +} +#[derive(Serialize, Debug)] +#[allow(missing_copy_implementations)] +pub struct WatchConfig { + pub local_displays_length: usize, +} +#[derive(Serialize, Debug)] +pub struct PathsConfig { + pub download_dir: PathBuf, + pub mpv_config_path: PathBuf, + pub mpv_input_path: PathBuf, + pub database_path: PathBuf, + pub last_selection_path: PathBuf, +} + +// pub fn status_path() -> anyhow::Result { +// const STATUS_PATH: &str = "running.info.json"; +// get_runtime_path(STATUS_PATH) +// } + +// pub fn subscriptions() -> anyhow::Result { +// const SUBSCRIPTIONS: &str = "subscriptions.json"; +// get_data_path(SUBSCRIPTIONS) +// } diff --git a/crates/yt/src/constants.rs b/crates/yt/src/constants.rs new file mode 100644 index 0000000..0f5b918 --- /dev/null +++ b/crates/yt/src/constants.rs @@ -0,0 +1,12 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +pub const HELP_STR: &str = include_str!("./select/selection_file/help.str"); diff --git a/crates/yt/src/download/download_options.rs b/crates/yt/src/download/download_options.rs new file mode 100644 index 0000000..03c20ba --- /dev/null +++ b/crates/yt/src/download/download_options.rs @@ -0,0 +1,118 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use anyhow::Context; +use serde_json::{Value, json}; +use yt_dlp::{YoutubeDL, YoutubeDLOptions}; + +use crate::{app::App, storage::video_database::YtDlpOptions}; + +use super::progress_hook::wrapped_progress_hook; + +pub fn download_opts(app: &App, additional_opts: &YtDlpOptions) -> anyhow::Result { + YoutubeDLOptions::new() + .with_progress_hook(wrapped_progress_hook) + .set("extract_flat", "in_playlist") + .set( + "extractor_args", + json! { + { + "youtube": { + "comment_sort": [ "top" ], + "max_comments": [ "150", "all", "100" ] + } + } + }, + ) + //.set("cookiesfrombrowser", json! {("firefox", "me.google", None::, "youtube_dlp")}) + .set("prefer_free_formats", true) + .set("ffmpeg_location", env!("FFMPEG_LOCATION")) + .set("format", "bestvideo[height<=?1080]+bestaudio/best") + .set("fragment_retries", 10) + .set("getcomments", true) + .set("ignoreerrors", false) + .set("retries", 10) + .set("writeinfojson", true) + // NOTE: This results in a constant warning message. <2025-01-04> + //.set("writeannotations", true) + .set("writesubtitles", true) + .set("writeautomaticsub", true) + .set( + "outtmpl", + json! { + { + "default": app.config.paths.download_dir.join("%(channel)s/%(title)s.%(ext)s"), + "chapter": "%(title)s - %(section_number)03d %(section_title)s [%(id)s].%(ext)s" + } + }, + ) + .set("compat_opts", json! {{}}) + .set("forceprint", json! {{}}) + .set("print_to_file", json! {{}}) + .set("windowsfilenames", false) + .set("restrictfilenames", false) + .set("trim_file_names", false) + .set( + "postprocessors", + json! { + [ + { + "api": "https://sponsor.ajay.app", + "categories": [ + "interaction", + "intro", + "music_offtopic", + "sponsor", + "outro", + "poi_highlight", + "preview", + "selfpromo", + "filler", + "chapter" + ], + "key": "SponsorBlock", + "when": "after_filter" + }, + { + "force_keyframes": false, + "key": "ModifyChapters", + "remove_chapters_patterns": [], + "remove_ranges": [], + "remove_sponsor_segments": [ "sponsor" ], + "sponsorblock_chapter_title": "[SponsorBlock]: %(category_names)l" + }, + { + "add_chapters": true, + "add_infojson": null, + "add_metadata": false, + "key": "FFmpegMetadata" + }, + { + "key": "FFmpegConcat", + "only_multi_video": true, + "when": "playlist" + } + ] + }, + ) + .set( + "subtitleslangs", + Value::Array( + additional_opts + .subtitle_langs + .split(',') + .map(|val| Value::String(val.to_owned())) + .collect::>(), + ), + ) + .build() + .context("Failed to instanciate download yt_dlp") +} diff --git a/crates/yt/src/download/mod.rs b/crates/yt/src/download/mod.rs new file mode 100644 index 0000000..110bf55 --- /dev/null +++ b/crates/yt/src/download/mod.rs @@ -0,0 +1,366 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::{collections::HashMap, io, str::FromStr, sync::Arc, time::Duration}; + +use crate::{ + app::App, + download::download_options::download_opts, + storage::video_database::{ + Video, YtDlpOptions, + downloader::{get_next_uncached_video, set_video_cache_path}, + extractor_hash::ExtractorHash, + get::get_video_yt_dlp_opts, + notify::wait_for_cache_reduction, + }, + unreachable::Unreachable, +}; + +use anyhow::{Context, Result, bail}; +use bytes::Bytes; +use futures::{FutureExt, future::BoxFuture}; +use log::{debug, error, info, warn}; +use tokio::{fs, task::JoinHandle, time}; +use yt_dlp::{json_cast, json_get}; + +#[allow(clippy::module_name_repetitions)] +pub mod download_options; +pub mod progress_hook; + +#[derive(Debug)] +#[allow(clippy::module_name_repetitions)] +pub struct CurrentDownload { + task_handle: JoinHandle>, + extractor_hash: ExtractorHash, +} + +impl CurrentDownload { + fn new_from_video(app: Arc, video: Video) -> Self { + let extractor_hash = video.extractor_hash; + + let task_handle = tokio::spawn(async move { + Downloader::actually_cache_video(&app, &video) + .await + .with_context(|| format!("Failed to cache video: '{}'", video.title))?; + Ok(()) + }); + + Self { + task_handle, + extractor_hash, + } + } +} + +enum CacheSizeCheck { + /// The video can be downloaded + Fits, + + /// The video and the current cache size together would exceed the size + TooLarge, + + /// The video would not even fit into the empty cache + ExceedsMaxCacheSize, +} + +#[derive(Debug)] +pub struct Downloader { + current_download: Option, + video_size_cache: HashMap, + printed_warning: bool, + cached_cache_allocation: Option, +} + +impl Default for Downloader { + fn default() -> Self { + Self::new() + } +} + +impl Downloader { + #[must_use] + pub fn new() -> Self { + Self { + current_download: None, + video_size_cache: HashMap::new(), + printed_warning: false, + cached_cache_allocation: None, + } + } + + /// Check if enough cache is available. Will wait for 10s if it's not. + async fn is_enough_cache_available( + &mut self, + app: &App, + max_cache_size: u64, + next_video: &Video, + ) -> Result { + if let Some(cdownload) = &self.current_download { + if cdownload.extractor_hash == next_video.extractor_hash { + // If the video is already being downloaded it will always fit. Otherwise the + // download would not have been started. + return Ok(CacheSizeCheck::Fits); + } + } + let cache_allocation = Self::get_current_cache_allocation(app).await?; + let video_size = self.get_approx_video_size(app, next_video)?; + + if video_size >= max_cache_size { + error!( + "The video '{}' ({}) exceeds the maximum cache size ({})! \ + Please set a bigger maximum (`--max-cache-size`) or skip it.", + next_video.title, + Bytes::new(video_size), + Bytes::new(max_cache_size) + ); + + return Ok(CacheSizeCheck::ExceedsMaxCacheSize); + } + + if cache_allocation.as_u64() + video_size >= max_cache_size { + if !self.printed_warning { + warn!( + "Can't download video: '{}' ({}) as it's too large for the cache ({} of {} allocated). \ + Waiting for cache size reduction..", + next_video.title, + Bytes::new(video_size), + &cache_allocation, + Bytes::new(max_cache_size) + ); + self.printed_warning = true; + + // Update this value immediately. + // This avoids printing the "Current cache size has changed .." warning below. + self.cached_cache_allocation = Some(cache_allocation); + } + + if let Some(cca) = self.cached_cache_allocation { + if cca != cache_allocation { + // Only print the warning if the display string has actually changed. + // Otherwise, we might confuse the user + if cca.to_string() != cache_allocation.to_string() { + warn!( + "Current cache size has changed, it's now: '{}'", + cache_allocation + ); + } + debug!( + "Cache size has changed: {} -> {}", + cca.as_u64(), + cache_allocation.as_u64() + ); + self.cached_cache_allocation = Some(cache_allocation); + } + } else { + unreachable!( + "The `printed_warning` should be false in this case, \ + and thus should have already set the `cached_cache_allocation`." + ); + } + + // Wait and hope, that a large video is deleted from the cache. + wait_for_cache_reduction(app).await?; + Ok(CacheSizeCheck::TooLarge) + } else { + self.printed_warning = false; + Ok(CacheSizeCheck::Fits) + } + } + + /// The entry point to the Downloader. + /// This Downloader will periodically check if the database has changed, and then also + /// change which videos it downloads. + /// This will run, until the database doesn't contain any watchable videos + pub async fn consume(&mut self, app: Arc, max_cache_size: u64) -> Result<()> { + while let Some(next_video) = get_next_uncached_video(&app).await? { + match self + .is_enough_cache_available(&app, max_cache_size, &next_video) + .await? + { + CacheSizeCheck::Fits => (), + CacheSizeCheck::TooLarge => continue, + CacheSizeCheck::ExceedsMaxCacheSize => bail!("Giving up."), + }; + + if self.current_download.is_some() { + let current_download = self.current_download.take().unreachable("It is `Some`."); + + if current_download.task_handle.is_finished() { + current_download.task_handle.await??; + continue; + } + + if next_video.extractor_hash == current_download.extractor_hash { + // Reset the taken value + self.current_download = Some(current_download); + } else { + info!( + "Noticed, that the next video is not the video being downloaded, replacing it ('{}' vs. '{}')!", + next_video.extractor_hash.into_short_hash(&app).await?, + current_download + .extractor_hash + .into_short_hash(&app) + .await? + ); + + // Replace the currently downloading video + // FIXME(@bpeetz): This does not work (probably because of the python part.) <2025-02-21> + current_download.task_handle.abort(); + + let new_current_download = + CurrentDownload::new_from_video(Arc::clone(&app), next_video); + + self.current_download = Some(new_current_download); + } + } else { + info!( + "No video is being downloaded right now, setting it to '{}'", + next_video.title + ); + let new_current_download = + CurrentDownload::new_from_video(Arc::clone(&app), next_video); + self.current_download = Some(new_current_download); + } + + // TODO(@bpeetz): Why do we sleep here? <2025-02-21> + time::sleep(Duration::from_secs(1)).await; + } + + info!("Finished downloading!"); + Ok(()) + } + + pub async fn get_current_cache_allocation(app: &App) -> Result { + fn dir_size(mut dir: fs::ReadDir) -> BoxFuture<'static, Result> { + async move { + let mut acc = 0; + while let Some(entry) = dir.next_entry().await? { + let size = match entry.metadata().await? { + data if data.is_dir() => { + let path = entry.path(); + let read_dir = fs::read_dir(path).await?; + + dir_size(read_dir).await?.as_u64() + } + data => data.len(), + }; + acc += size; + } + Ok(Bytes::new(acc)) + } + .boxed() + } + + let read_dir_result = match fs::read_dir(&app.config.paths.download_dir).await { + Ok(ok) => ok, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => { + fs::create_dir_all(&app.config.paths.download_dir) + .await + .with_context(|| { + format!( + "Failed to create download dir at: '{}'", + &app.config.paths.download_dir.display() + ) + })?; + + info!( + "Created empty download dir at '{}'", + &app.config.paths.download_dir.display(), + ); + + // The new dir should not contain anything (otherwise we would not have had to + // create it) + return Ok(Bytes::new(0)); + } + err => Err(io::Error::from(err)).with_context(|| { + format!( + "Failed to get dir size of download dir at: '{}'", + &app.config.paths.download_dir.display() + ) + })?, + }, + }; + + dir_size(read_dir_result).await + } + + fn get_approx_video_size(&mut self, app: &App, video: &Video) -> Result { + if let Some(value) = self.video_size_cache.get(&video.extractor_hash) { + Ok(*value) + } else { + // the subtitle file size should be negligible + let add_opts = YtDlpOptions { + subtitle_langs: String::new(), + }; + let yt_dlp = download_opts(app, &add_opts)?; + + let result = yt_dlp + .extract_info(&video.url, false, true) + .with_context(|| { + format!("Failed to extract video information: '{}'", video.title) + })?; + + let size = if let Some(val) = result.get("filesize") { + json_cast!(val, as_u64) + } else if let Some(val) = result.get("filesize_approx") { + json_cast!(val, as_u64) + } else if result.get("duration").is_some() && result.get("tbr").is_some() { + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let duration = json_get!(result, "duration", as_f64).ceil() as u64; + + // TODO: yt_dlp gets this from the format + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let tbr = json_get!(result, "tbr", as_f64).ceil() as u64; + + duration * tbr * (1000 / 8) + } else { + let hardcoded_default = Bytes::from_str("250 MiB").expect("This is hardcoded"); + error!( + "Failed to find a filesize for video: '{}' (Using hardcoded value of {})", + video.title, hardcoded_default + ); + hardcoded_default.as_u64() + }; + + assert_eq!( + self.video_size_cache.insert(video.extractor_hash, size), + None + ); + + Ok(size) + } + } + + async fn actually_cache_video(app: &App, video: &Video) -> Result<()> { + debug!("Download started: {}", &video.title); + + let addional_opts = get_video_yt_dlp_opts(app, &video.extractor_hash).await?; + let yt_dlp = download_opts(app, &addional_opts)?; + + let result = yt_dlp + .download(&[video.url.to_owned()]) + .with_context(|| format!("Failed to download video: '{}'", video.title))?; + + assert_eq!(result.len(), 1); + let result = &result[0]; + + set_video_cache_path(app, &video.extractor_hash, Some(result)).await?; + + info!( + "Video '{}' was downlaoded to path: {}", + video.title, + result.display() + ); + + Ok(()) + } +} diff --git a/crates/yt/src/download/progress_hook.rs b/crates/yt/src/download/progress_hook.rs new file mode 100644 index 0000000..b75ec00 --- /dev/null +++ b/crates/yt/src/download/progress_hook.rs @@ -0,0 +1,188 @@ +use std::{ + io::{Write, stderr}, + process, +}; + +use bytes::Bytes; +use log::{Level, log_enabled}; +use yt_dlp::mk_python_function; + +use crate::{ + ansi_escape_codes::{clear_whole_line, move_to_col}, + select::selection_file::duration::MaybeDuration, +}; + +/// # Panics +/// If expectations fail. +#[allow(clippy::too_many_lines, clippy::needless_pass_by_value)] +pub fn progress_hook( + input: serde_json::Map, +) -> Result<(), std::io::Error> { + // Only add the handler, if the log-level is higher than Debug (this avoids covering debug + // messages). + if log_enabled!(Level::Debug) { + return Ok(()); + } + + macro_rules! get { + (@interrogate $item:ident, $type_fun:ident, $get_fun:ident, $name:expr) => {{ + let a = $item.get($name).expect(concat!( + "The field '", + stringify!($name), + "' should exist." + )); + + if a.$type_fun() { + a.$get_fun().expect( + "The should have been checked in the if guard, so unpacking here is fine", + ) + } else { + panic!( + "Value {} => \n{}\n is not of type: {}", + $name, + a, + stringify!($type_fun) + ); + } + }}; + + ($type_fun:ident, $get_fun:ident, $name1:expr, $name2:expr) => {{ + let a = get! {@interrogate input, is_object, as_object, $name1}; + let b = get! {@interrogate a, $type_fun, $get_fun, $name2}; + b + }}; + + ($type_fun:ident, $get_fun:ident, $name:expr) => {{ + get! {@interrogate input, $type_fun, $get_fun, $name} + }}; + } + + macro_rules! default_get { + (@interrogate $item:ident, $default:expr, $get_fun:ident, $name:expr) => {{ + let a = if let Some(field) = $item.get($name) { + field.$get_fun().unwrap_or($default) + } else { + $default + }; + a + }}; + + ($get_fun:ident, $default:expr, $name1:expr, $name2:expr) => {{ + let a = get! {@interrogate input, is_object, as_object, $name1}; + let b = default_get! {@interrogate a, $default, $get_fun, $name2}; + b + }}; + + ($get_fun:ident, $default:expr, $name:expr) => {{ + default_get! {@interrogate input, $default, $get_fun, $name} + }}; + } + + macro_rules! c { + ($color:expr, $format:expr) => { + format!("\x1b[{}m{}\x1b[0m", $color, $format) + }; + } + + #[allow(clippy::items_after_statements)] + fn format_bytes(bytes: u64) -> String { + let bytes = Bytes::new(bytes); + bytes.to_string() + } + + #[allow(clippy::items_after_statements)] + fn format_speed(speed: f64) -> String { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let bytes = Bytes::new(speed.floor() as u64); + format!("{bytes}/s") + } + + let get_title = || -> String { + match get! {is_string, as_str, "info_dict", "ext"} { + "vtt" => { + format!( + "Subtitles ({})", + default_get! {as_str, "", "info_dict", "name"} + ) + } + "webm" | "mp4" | "mp3" | "m4a" => { + default_get! { as_str, "", "info_dict", "title"}.to_owned() + } + other => panic!("The extension '{other}' is not yet implemented"), + } + }; + + match get! {is_string, as_str, "status"} { + "downloading" => { + let elapsed = default_get! {as_f64, 0.0f64, "elapsed"}; + let eta = default_get! {as_f64, 0.0, "eta"}; + let speed = default_get! {as_f64, 0.0, "speed"}; + + let downloaded_bytes = get! {is_u64, as_u64, "downloaded_bytes"}; + let (total_bytes, bytes_is_estimate): (u64, &'static str) = { + let total_bytes = default_get!(as_u64, 0, "total_bytes"); + if total_bytes == 0 { + let maybe_estimate = default_get!(as_u64, 0, "total_bytes_estimate"); + + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + if maybe_estimate == 0 { + // The download speed should be in bytes per second and the eta in seconds. + // Thus multiplying them gets us the raw bytes (which were estimated by `yt_dlp`, from their `info.json`) + let bytes_still_needed = (speed * eta).ceil() as u64; + + (downloaded_bytes + bytes_still_needed, "~") + } else { + (maybe_estimate, "~") + } + } else { + (total_bytes, "") + } + }; + + let percent: f64 = { + if total_bytes == 0 { + 100.0 + } else { + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::cast_precision_loss + )] + { + (downloaded_bytes as f64 / total_bytes as f64) * 100.0 + } + } + }; + + clear_whole_line(); + move_to_col(1); + + eprint!( + "'{}' [{}/{} at {}] -> [{} of {}{} {}] ", + c!("34;1", get_title()), + c!("33;1", MaybeDuration::from_secs_f64(elapsed)), + c!("33;1", MaybeDuration::from_secs_f64(eta)), + c!("32;1", format_speed(speed)), + c!("31;1", format_bytes(downloaded_bytes)), + c!("31;1", bytes_is_estimate), + c!("31;1", format_bytes(total_bytes)), + c!("36;1", format!("{:.02}%", percent)) + ); + stderr().flush()?; + } + "finished" => { + eprintln!("-> Finished downloading."); + } + "error" => { + // TODO: This should probably return an Err. But I'm not so sure where the error would + // bubble up to (i.e., who would catch it) <2025-01-21> + eprintln!("-> Error while downloading: {}", get_title()); + process::exit(1); + } + other => unreachable!("'{other}' should not be a valid state!"), + } + + Ok(()) +} + +mk_python_function!(progress_hook, wrapped_progress_hook); diff --git a/crates/yt/src/main.rs b/crates/yt/src/main.rs new file mode 100644 index 0000000..39f52f4 --- /dev/null +++ b/crates/yt/src/main.rs @@ -0,0 +1,247 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +// `yt` is not a library. Besides, the `anyhow::Result` type is really useless, if you're not going +// to print it anyways. +#![allow(clippy::missing_errors_doc)] + +use std::sync::Arc; + +use anyhow::{Context, Result, bail}; +use app::App; +use bytes::Bytes; +use cache::{invalidate, maintain}; +use clap::Parser; +use cli::{CacheCommand, SelectCommand, SubscriptionCommand, VideosCommand}; +use config::Config; +use log::{error, info}; +use select::cmds::handle_select_cmd; +use storage::video_database::get::video_by_hash; +use tokio::{ + fs::File, + io::{BufReader, stdin}, + task::JoinHandle, +}; + +use crate::{cli::Command, storage::subscriptions}; + +pub mod ansi_escape_codes; +pub mod app; +pub mod cli; +pub mod unreachable; + +pub mod cache; +pub mod comments; +pub mod config; +pub mod constants; +pub mod download; +pub mod select; +pub mod status; +pub mod storage; +pub mod subscribe; +pub mod update; +pub mod version; +pub mod videos; +pub mod watch; + +#[tokio::main] +// This is _the_ main function after all. It is not really good, but it sort of works. +#[allow(clippy::too_many_lines)] +async fn main() -> Result<()> { + let args = cli::CliArgs::parse(); + + // The default verbosity is 1 (Warn) + let verbosity: u8 = args.verbosity + 1; + + stderrlog::new() + .module(module_path!()) + .modules(&["yt_dlp".to_owned(), "libmpv2".to_owned()]) + .quiet(args.quiet) + .show_module_names(false) + .color(stderrlog::ColorChoice::Auto) + .verbosity(verbosity as usize) + .timestamp(stderrlog::Timestamp::Off) + .init() + .expect("Let's just hope that this does not panic"); + + info!("Using verbosity level: '{} ({})'", verbosity, { + match verbosity { + 0 => "Error", + 1 => "Warn", + 2 => "Info", + 3 => "Debug", + 4.. => "Trace", + } + }); + + let config = Config::from_config_file(args.db_path, args.config_path, args.color)?; + if args.version { + version::show(&config).await?; + return Ok(()); + } + + let app = App::new(config, !args.no_migrate_db).await?; + + match args.command.unwrap_or(Command::default()) { + Command::Download { + force, + max_cache_size, + } => { + let max_cache_size = + max_cache_size.unwrap_or(app.config.download.max_cache_size.as_u64()); + info!("Max cache size: '{}'", Bytes::new(max_cache_size)); + + maintain(&app, false).await?; + if force { + invalidate(&app, true).await?; + } + + download::Downloader::new() + .consume(Arc::new(app), max_cache_size) + .await?; + } + Command::Select { cmd } => { + let cmd = cmd.unwrap_or(SelectCommand::default()); + + match cmd { + SelectCommand::File { + done, + use_last_selection, + } => Box::pin(select::select(&app, done, use_last_selection)).await?, + _ => Box::pin(handle_select_cmd(&app, cmd, None)).await?, + } + } + Command::Sedowa {} => { + Box::pin(select::select(&app, false, false)).await?; + + let arc_app = Arc::new(app); + dowa(arc_app).await?; + } + Command::Dowa {} => { + let arc_app = Arc::new(app); + dowa(arc_app).await?; + } + Command::Videos { cmd } => match cmd { + VideosCommand::List { + search_query, + limit, + } => { + videos::query(&app, limit, search_query) + .await + .context("Failed to query videos")?; + } + VideosCommand::Info { hash } => { + let video = video_by_hash(&app, &hash.realize(&app).await?).await?; + + print!( + "{}", + &video + .to_info_display(&app) + .await + .context("Failed to format video")? + ); + } + }, + Command::Update { + max_backlog, + subscriptions, + } => { + let all_subs = subscriptions::get(&app).await?; + + for sub in &subscriptions { + if !all_subs.0.contains_key(sub) { + bail!( + "Your specified subscription to update '{}' is not a subscription!", + sub + ) + } + } + + let max_backlog = max_backlog.unwrap_or(app.config.update.max_backlog); + + update::update(&app, max_backlog, subscriptions).await?; + } + Command::Subscriptions { cmd } => match cmd { + SubscriptionCommand::Add { name, url } => { + subscribe::subscribe(&app, name, url) + .await + .context("Failed to add a subscription")?; + } + SubscriptionCommand::Remove { name } => { + subscribe::unsubscribe(&app, name) + .await + .context("Failed to remove a subscription")?; + } + SubscriptionCommand::List {} => { + let all_subs = subscriptions::get(&app).await?; + + for (key, val) in all_subs.0 { + println!("{}: '{}'", key, val.url); + } + } + SubscriptionCommand::Export {} => { + let all_subs = subscriptions::get(&app).await?; + for val in all_subs.0.values() { + println!("{}", val.url); + } + } + SubscriptionCommand::Import { file, force } => { + if let Some(file) = file { + let f = File::open(file).await?; + + subscribe::import(&app, BufReader::new(f), force).await?; + } else { + subscribe::import(&app, BufReader::new(stdin()), force).await?; + } + } + }, + + Command::Watch {} => watch::watch(Arc::new(app)).await?, + Command::Playlist { watch } => watch::playlist::playlist(&app, watch).await?, + + Command::Status {} => status::show(&app).await?, + Command::Config {} => status::config(&app)?, + + Command::Database { command } => match command { + CacheCommand::Invalidate { hard } => invalidate(&app, hard).await?, + CacheCommand::Maintain { all } => maintain(&app, all).await?, + }, + + Command::Comments {} => { + comments::comments(&app).await?; + } + Command::Description {} => { + comments::description(&app).await?; + } + } + + Ok(()) +} + +async fn dowa(arc_app: Arc) -> Result<()> { + let max_cache_size = arc_app.config.download.max_cache_size; + info!("Max cache size: '{}'", max_cache_size); + + let arc_app_clone = Arc::clone(&arc_app); + let download: JoinHandle<()> = tokio::spawn(async move { + let result = download::Downloader::new() + .consume(arc_app_clone, max_cache_size.as_u64()) + .await; + + if let Err(err) = result { + error!("Error from downloader: {err:?}"); + } + }); + + watch::watch(arc_app).await?; + download.await?; + Ok(()) +} diff --git a/crates/yt/src/select/cmds/add.rs b/crates/yt/src/select/cmds/add.rs new file mode 100644 index 0000000..387b3a1 --- /dev/null +++ b/crates/yt/src/select/cmds/add.rs @@ -0,0 +1,191 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use crate::{ + app::App, + download::download_options::download_opts, + storage::video_database::{ + self, extractor_hash::ExtractorHash, get::get_all_hashes, set::add_video, + }, + update::video_entry_to_video, +}; + +use anyhow::{Context, Result, bail}; +use log::{error, warn}; +use url::Url; +use yt_dlp::{InfoJson, YoutubeDL, json_cast, json_get}; + +#[allow(clippy::too_many_lines)] +pub(super) async fn add( + app: &App, + urls: Vec, + start: Option, + stop: Option, +) -> Result<()> { + for url in urls { + async fn process_and_add(app: &App, entry: InfoJson, yt_dlp: &YoutubeDL) -> Result<()> { + let url = json_get!(entry, "url", as_str).parse()?; + + let entry = yt_dlp + .extract_info(&url, false, true) + .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; + + add_entry(app, entry).await?; + + Ok(()) + } + + async fn add_entry(app: &App, entry: InfoJson) -> Result<()> { + // We have to re-fetch all hashes every time, because a user could try to add the same + // URL twice (for whatever reason.) + let hashes = get_all_hashes(app) + .await + .context("Failed to fetch all video hashes")?; + let extractor_hash = blake3::hash(json_get!(entry, "id", as_str).as_bytes()); + if hashes.contains(&extractor_hash) { + error!( + "Video '{}'{} is already in the database. Skipped adding it", + ExtractorHash::from_hash(extractor_hash) + .into_short_hash(app) + .await + .with_context(|| format!( + "Failed to format hash of video '{}' as short hash", + entry + .get("url") + .map_or("".to_owned(), ToString::to_string) + ))?, + entry + .get("title") + .map_or(String::new(), |title| format!(" ('{title}')")) + ); + return Ok(()); + } + + let video = video_entry_to_video(&entry, None)?; + add_video(app, video.clone()).await?; + + println!("{}", &video.to_line_display(app).await?); + + Ok(()) + } + + let yt_dlp = download_opts( + app, + &video_database::YtDlpOptions { + subtitle_langs: String::new(), + }, + )?; + + let entry = yt_dlp + .extract_info(&url, false, true) + .with_context(|| format!("Failed to fetch entry for url: '{url}'"))?; + + match entry.get("_type").map(|val| json_cast!(val, as_str)) { + Some("Video") => { + add_entry(app, entry).await?; + if start.is_some() || stop.is_some() { + warn!( + "You added `start` and/or `stop` markers for a single *video*! These will be ignored." + ); + } + } + Some("Playlist") => { + if let Some(entries) = entry.get("entries") { + let entries = json_cast!(entries, as_array); + let start = start.unwrap_or(0); + let stop = stop.unwrap_or(entries.len() - 1); + + let respected_entries = + take_vector(entries, start, stop).with_context(|| { + format!( + "Failed to take entries starting at: {start} and ending with {stop}" + ) + })?; + + if respected_entries.is_empty() { + warn!("No entries found, after applying your start/stop limits."); + } else { + // Pre-warm the cache + process_and_add( + app, + json_cast!(respected_entries[0], as_object).to_owned(), + &yt_dlp, + ) + .await?; + let respected_entries = &respected_entries[1..]; + + let futures: Vec<_> = respected_entries + .iter() + .map(|entry| { + process_and_add( + app, + json_cast!(entry, as_object).to_owned(), + &yt_dlp, + ) + }) + .collect(); + + for fut in futures { + fut.await?; + } + } + } else { + bail!("Your playlist does not seem to have any entries!") + } + } + other => bail!( + "Your URL should point to a video or a playlist, but points to a '{:#?}'", + other + ), + } + } + + Ok(()) +} + +fn take_vector(vector: &[T], start: usize, stop: usize) -> Result<&[T]> { + let length = vector.len(); + + if stop >= length { + bail!( + "Your stop marker ({stop}) exceeds the possible entries ({length})! Remember that it is zero indexed." + ); + } + + Ok(&vector[start..=stop]) +} + +#[cfg(test)] +mod test { + use crate::select::cmds::add::take_vector; + + #[test] + fn test_vector_take() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + let new_vec = take_vector(&vec, 2, 8).unwrap(); + + assert_eq!(new_vec, vec![2, 3, 4, 5, 6, 7, 8]); + } + + #[test] + fn test_vector_take_overflow() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + assert!(take_vector(&vec, 0, 12).is_err()); + } + + #[test] + fn test_vector_take_equal() { + let vec = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + assert!(take_vector(&vec, 0, 11).is_err()); + } +} diff --git a/crates/yt/src/select/cmds/mod.rs b/crates/yt/src/select/cmds/mod.rs new file mode 100644 index 0000000..aabcd3d --- /dev/null +++ b/crates/yt/src/select/cmds/mod.rs @@ -0,0 +1,111 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use crate::{ + app::App, + cli::{SelectCommand, SharedSelectionCommandArgs}, + storage::video_database::{ + Priority, VideoOptions, VideoStatus, + get::video_by_hash, + set::{set_video_options, video_status}, + }, +}; + +use anyhow::{Context, Result, bail}; + +mod add; + +pub async fn handle_select_cmd( + app: &App, + cmd: SelectCommand, + line_number: Option, +) -> Result<()> { + match cmd { + SelectCommand::Pick { shared } => { + handle_status_change(app, shared, line_number, VideoStatus::Pick).await?; + } + SelectCommand::Drop { shared } => { + handle_status_change(app, shared, line_number, VideoStatus::Drop).await?; + } + SelectCommand::Watched { shared } => { + handle_status_change(app, shared, line_number, VideoStatus::Watched).await?; + } + SelectCommand::Add { urls, start, stop } => { + Box::pin(add::add(app, urls, start, stop)).await?; + } + SelectCommand::Watch { shared } => { + let hash = shared.hash.clone().realize(app).await?; + + let video = video_by_hash(app, &hash).await?; + + if let VideoStatus::Cached { + cache_path, + is_focused, + } = video.status + { + handle_status_change( + app, + shared, + line_number, + VideoStatus::Cached { + cache_path, + is_focused, + }, + ) + .await?; + } else { + handle_status_change(app, shared, line_number, VideoStatus::Watch).await?; + } + } + + SelectCommand::Url { shared } => { + let Some(url) = shared.url else { + bail!("You need to provide a url to `select url ..`") + }; + + let mut firefox = std::process::Command::new("firefox"); + firefox.args(["-P", "timesinks.youtube"]); + firefox.arg(url.as_str()); + let _handle = firefox.spawn().context("Failed to run firefox")?; + } + SelectCommand::File { .. } => unreachable!("This should have been filtered out"), + } + Ok(()) +} + +async fn handle_status_change( + app: &App, + shared: SharedSelectionCommandArgs, + line_number: Option, + new_status: VideoStatus, +) -> Result<()> { + let hash = shared.hash.realize(app).await?; + let video_options = VideoOptions::new( + shared + .subtitle_langs + .unwrap_or(app.config.select.subtitle_langs.clone()), + shared.speed.unwrap_or(app.config.select.playback_speed), + ); + let priority = compute_priority(line_number, shared.priority); + + video_status(app, &hash, new_status, priority).await?; + set_video_options(app, &hash, &video_options).await?; + + Ok(()) +} + +fn compute_priority(line_number: Option, priority: Option) -> Option { + if let Some(pri) = priority { + Some(Priority::from(pri)) + } else { + line_number.map(Priority::from) + } +} diff --git a/crates/yt/src/select/mod.rs b/crates/yt/src/select/mod.rs new file mode 100644 index 0000000..8db9ae3 --- /dev/null +++ b/crates/yt/src/select/mod.rs @@ -0,0 +1,176 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::{ + env::{self}, + fs, + io::{BufRead, BufReader, BufWriter, Write}, + string::String, +}; + +use crate::{ + app::App, + cli::CliArgs, + constants::HELP_STR, + storage::video_database::{Video, VideoStatusMarker, get}, + unreachable::Unreachable, +}; + +use anyhow::{Context, Result, bail}; +use clap::Parser; +use cmds::handle_select_cmd; +use futures::{TryStreamExt, stream::FuturesOrdered}; +use selection_file::process_line; +use tempfile::Builder; +use tokio::process::Command; + +pub mod cmds; +pub mod selection_file; + +async fn to_select_file_display_owned(video: Video, app: &App) -> Result { + video.to_select_file_display(app).await +} + +pub async fn select(app: &App, done: bool, use_last_selection: bool) -> Result<()> { + let temp_file = Builder::new() + .prefix("yt_video_select-") + .suffix(".yts") + .rand_bytes(6) + .tempfile() + .context("Failed to get tempfile")?; + + if use_last_selection { + fs::copy(&app.config.paths.last_selection_path, &temp_file)?; + } else { + let matching_videos = if done { + get::videos(app, VideoStatusMarker::ALL).await? + } else { + get::videos( + app, + &[ + VideoStatusMarker::Pick, + // + VideoStatusMarker::Watch, + VideoStatusMarker::Cached, + ], + ) + .await? + }; + + // Warmup the cache for the display rendering of the videos. + // Otherwise the futures would all try to warm it up at the same time. + if let Some(vid) = matching_videos.first() { + drop(vid.to_line_display(app).await?); + } + + let mut edit_file = BufWriter::new(&temp_file); + + matching_videos + .into_iter() + .map(|vid| to_select_file_display_owned(vid, app)) + .collect::>() + .try_collect::>() + .await? + .into_iter() + .try_for_each(|line| -> Result<()> { + edit_file + .write_all(line.as_bytes()) + .context("Failed to write to `edit_file`")?; + + Ok(()) + })?; + + edit_file.write_all(HELP_STR.as_bytes())?; + edit_file.flush().context("Failed to flush edit file")?; + }; + + { + let editor = env::var("EDITOR").unwrap_or("nvim".to_owned()); + + let mut nvim = Command::new(editor); + nvim.arg(temp_file.path()); + let status = nvim.status().await.context("Falied to run nvim")?; + if !status.success() { + bail!("nvim exited with error status: {}", status) + } + } + + let read_file = temp_file.reopen()?; + fs::copy(temp_file.path(), &app.config.paths.last_selection_path) + .context("Failed to persist selection file")?; + + let reader = BufReader::new(&read_file); + + let mut line_number = 0; + for line in reader.lines() { + let line = line.context("Failed to read a line")?; + + if let Some(line) = process_line(&line)? { + line_number -= 1; + + // debug!( + // "Parsed command: `{}`", + // line.iter() + // .map(|val| format!("\"{}\"", val)) + // .collect::>() + // .join(" ") + // ); + + let arg_line = ["yt", "select"] + .into_iter() + .chain(line.iter().map(String::as_str)); + + let args = CliArgs::parse_from(arg_line); + + let crate::cli::Command::Select { cmd } = args + .command + .unreachable("This will be some, as we constructed it above.") + else { + unreachable!("This is checked in the `filter_line` function") + }; + + Box::pin(handle_select_cmd( + app, + cmd.unreachable( + "This value should always be some \ + here, as it would otherwise thrown an error above.", + ), + Some(line_number), + )) + .await?; + } + } + + Ok(()) +} + +// // FIXME: There should be no reason why we need to re-run yt, just to get the help string. But I've +// // yet to find a way to do it without the extra exec <2024-08-20> +// async fn get_help() -> Result { +// let binary_name = current_exe()?; +// let cmd = Command::new(binary_name) +// .args(&["select", "--help"]) +// .output() +// .await?; +// +// assert_eq!(cmd.status.code(), Some(0)); +// +// let output = String::from_utf8(cmd.stdout).expect("Our help output was not utf8?"); +// +// let out = output +// .lines() +// .map(|line| format!("# {}\n", line)) +// .collect::(); +// +// debug!("Returning help: '{}'", &out); +// +// Ok(out) +// } diff --git a/crates/yt/src/select/selection_file/duration.rs b/crates/yt/src/select/selection_file/duration.rs new file mode 100644 index 0000000..77c4fc5 --- /dev/null +++ b/crates/yt/src/select/selection_file/duration.rs @@ -0,0 +1,185 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::str::FromStr; +use std::time::Duration; + +use anyhow::{Context, Result}; + +const SECOND: u64 = 1; +const MINUTE: u64 = 60 * SECOND; +const HOUR: u64 = 60 * MINUTE; +const DAY: u64 = 24 * HOUR; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct MaybeDuration { + time: Option, +} + +impl MaybeDuration { + #[must_use] + pub fn from_std(d: Duration) -> Self { + Self { time: Some(d) } + } + + #[must_use] + pub fn from_secs_f64(d: f64) -> Self { + Self { + time: Some(Duration::from_secs_f64(d)), + } + } + #[must_use] + pub fn from_maybe_secs_f64(d: Option) -> Self { + Self { + time: d.map(Duration::from_secs_f64), + } + } + #[must_use] + pub fn from_secs(d: u64) -> Self { + Self { + time: Some(Duration::from_secs(d)), + } + } + + #[must_use] + pub fn zero() -> Self { + Self { + time: Some(Duration::default()), + } + } + + /// Try to return the current duration encoded as seconds. + #[must_use] + pub fn as_secs(&self) -> Option { + self.time.map(|v| v.as_secs()) + } + + /// Try to return the current duration encoded as seconds and nanoseconds. + #[must_use] + pub fn as_secs_f64(&self) -> Option { + self.time.map(|v| v.as_secs_f64()) + } +} + +impl FromStr for MaybeDuration { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + fn parse_num(str: &str, suffix: char) -> Result { + str.strip_suffix(suffix) + .with_context(|| format!("Failed to strip suffix '{suffix}' of number: '{str}'"))? + .parse::() + .with_context(|| format!("Failed to parse '{suffix}'")) + } + + if s == "[No duration]" { + return Ok(Self { time: None }); + } + + let buf: Vec<_> = s.split(' ').collect(); + + let days; + let hours; + let minutes; + let seconds; + + assert_eq!(buf.len(), 2, "Other lengths should not happen"); + + if buf[0].ends_with('d') { + days = parse_num(buf[0], 'd')?; + hours = parse_num(buf[1], 'h')?; + minutes = parse_num(buf[2], 'm')?; + seconds = 0; + } else if buf[0].ends_with('h') { + days = 0; + hours = parse_num(buf[0], 'h')?; + minutes = parse_num(buf[1], 'm')?; + seconds = 0; + } else if buf[0].ends_with('m') { + days = 0; + hours = 0; + minutes = parse_num(buf[0], 'm')?; + seconds = parse_num(buf[1], 's')?; + } else { + unreachable!( + "The first part always ends with 'h' or 'm', but was: {:#?}", + buf + ) + } + + Ok(Self { + time: Some(Duration::from_secs( + days * DAY + hours * HOUR + minutes * MINUTE + seconds * SECOND, + )), + }) + } +} + +impl std::fmt::Display for MaybeDuration { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + if let Some(self_seconds) = self.as_secs() { + let base_day = self_seconds - (self_seconds % DAY); + let base_hour = (self_seconds % DAY) - ((self_seconds % DAY) % HOUR); + let base_min = (self_seconds % HOUR) - (((self_seconds % DAY) % HOUR) % MINUTE); + let base_sec = ((self_seconds % DAY) % HOUR) % MINUTE; + + let d = base_day / DAY; + let h = base_hour / HOUR; + let m = base_min / MINUTE; + let s = base_sec / SECOND; + + if d > 0 { + write!(fmt, "{d}d {h}h {m}m") + } else if h > 0 { + write!(fmt, "{h}h {m}m") + } else { + write!(fmt, "{m}m {s}s") + } + } else { + write!(fmt, "[No duration]") + } + } +} +#[cfg(test)] +mod test { + use std::str::FromStr; + + use crate::select::selection_file::duration::{DAY, HOUR, MINUTE}; + + use super::MaybeDuration; + + #[test] + fn test_display_duration_1h() { + let dur = MaybeDuration::from_secs(HOUR); + assert_eq!("1h 0m".to_owned(), dur.to_string()); + } + #[test] + fn test_display_duration_30min() { + let dur = MaybeDuration::from_secs(MINUTE * 30); + assert_eq!("30m 0s".to_owned(), dur.to_string()); + } + #[test] + fn test_display_duration_1d() { + let dur = MaybeDuration::from_secs(DAY + MINUTE * 30 + HOUR * 2); + assert_eq!("1d 2h 30m".to_owned(), dur.to_string()); + } + + #[test] + fn test_display_duration_roundtrip() { + let dur = MaybeDuration::zero(); + let dur_str = dur.to_string(); + + assert_eq!( + MaybeDuration::zero(), + MaybeDuration::from_str(&dur_str).unwrap() + ); + } +} diff --git a/crates/yt/src/select/selection_file/help.str b/crates/yt/src/select/selection_file/help.str new file mode 100644 index 0000000..e3cc347 --- /dev/null +++ b/crates/yt/src/select/selection_file/help.str @@ -0,0 +1,12 @@ +# Commands: +# w, watch [-p,-s,-l] Mark the video given by the hash to be watched +# wd, watched [-p,-s,-l] Mark the video given by the hash as already watched +# d, drop [-p,-s,-l] Mark the video given by the hash to be dropped +# u, url [-p,-s,-l] Open the video URL in Firefox's `timesinks.youtube` profile +# p, pick [-p,-s,-l] Reset the videos status to 'Pick' +# a, add URL Add a video, defined by the URL +# +# See `yt select --help` for more help. +# +# These lines can be re-ordered; they are executed from top to bottom. +# vim: filetype=yts conceallevel=2 concealcursor=nc colorcolumn= nowrap diff --git a/crates/yt/src/select/selection_file/help.str.license b/crates/yt/src/select/selection_file/help.str.license new file mode 100644 index 0000000..a0e196c --- /dev/null +++ b/crates/yt/src/select/selection_file/help.str.license @@ -0,0 +1,10 @@ +yt - A fully featured command line YouTube client + +Copyright (C) 2024 Benedikt Peetz +Copyright (C) 2025 Benedikt Peetz +SPDX-License-Identifier: GPL-3.0-or-later + +This file is part of Yt. + +You should have received a copy of the License along with this program. +If not, see . diff --git a/crates/yt/src/select/selection_file/mod.rs b/crates/yt/src/select/selection_file/mod.rs new file mode 100644 index 0000000..abd26c4 --- /dev/null +++ b/crates/yt/src/select/selection_file/mod.rs @@ -0,0 +1,32 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +//! The data structures needed to express the file, which the user edits + +use anyhow::{Context, Result}; +use trinitry::Trinitry; + +pub mod duration; + +pub fn process_line(line: &str) -> Result>> { + // Filter out comments and empty lines + if line.starts_with('#') || line.trim().is_empty() { + Ok(None) + } else { + let tri = Trinitry::new(line).with_context(|| format!("Failed to parse line '{line}'"))?; + + let mut vec = Vec::with_capacity(tri.arguments().len() + 1); + vec.push(tri.command().to_owned()); + vec.extend(tri.arguments().to_vec()); + + Ok(Some(vec)) + } +} diff --git a/crates/yt/src/status/mod.rs b/crates/yt/src/status/mod.rs new file mode 100644 index 0000000..18bef7d --- /dev/null +++ b/crates/yt/src/status/mod.rs @@ -0,0 +1,129 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::time::Duration; + +use crate::{ + app::App, + download::Downloader, + select::selection_file::duration::MaybeDuration, + storage::{ + subscriptions, + video_database::{VideoStatusMarker, get}, + }, +}; + +use anyhow::{Context, Result}; +use bytes::Bytes; + +macro_rules! get { + ($videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .count() + }; + + (@collect $videos:expr, $status:ident) => { + $videos + .iter() + .filter(|vid| vid.status.as_marker() == VideoStatusMarker::$status) + .collect() + }; +} + +pub async fn show(app: &App) -> Result<()> { + let all_videos = get::videos(app, VideoStatusMarker::ALL).await?; + + // lengths + let picked_videos_len = get!(all_videos, Pick); + + let watch_videos_len = get!(all_videos, Watch); + let cached_videos_len = get!(all_videos, Cached); + let watched_videos_len = get!(all_videos, Watched); + let watched_videos: Vec<_> = get!(@collect all_videos, Watched); + + let drop_videos_len = get!(all_videos, Drop); + let dropped_videos_len = get!(all_videos, Dropped); + + let subscriptions = subscriptions::get(app).await?; + let subscriptions_len = subscriptions.0.len(); + + let watchtime_status = { + let total_watch_time_raw = watched_videos + .iter() + .fold(Duration::default(), |acc, vid| acc + vid.watch_progress); + + // Most things are watched at a speed of s (which is defined in the config file). + // Thus + // y = x * s -> y / s = x + let total_watch_time = Duration::from_secs_f64( + (total_watch_time_raw.as_secs_f64()) / app.config.select.playback_speed, + ); + + let speed = app.config.select.playback_speed; + + // Do not print the adjusted time, if the user has keep the speed level at 1. + #[allow(clippy::float_cmp)] + if speed == 1.0 { + format!( + "Total Watchtime: {}\n", + MaybeDuration::from_std(total_watch_time_raw) + ) + } else { + format!( + "Total Watchtime: {} (at {speed} speed: {})\n", + MaybeDuration::from_std(total_watch_time_raw), + MaybeDuration::from_std(total_watch_time), + ) + } + }; + + let watch_rate: f64 = { + fn to_f64(input: usize) -> f64 { + f64::from(u32::try_from(input).expect("This should never exceed u32::MAX")) + } + + let count = to_f64(watched_videos_len) / (to_f64(drop_videos_len) + to_f64(dropped_videos_len)); + count * 100.0 + }; + + let cache_usage_raw = Downloader::get_current_cache_allocation(app) + .await + .context("Failed to get current cache allocation")?; + let cache_usage: Bytes = cache_usage_raw; + println!( + "\ +Picked Videos: {picked_videos_len} + +Watch Videos: {watch_videos_len} +Cached Videos: {cached_videos_len} +Watched Videos: {watched_videos_len} (watch rate: {watch_rate:.2} %) + +Drop Videos: {drop_videos_len} +Dropped Videos: {dropped_videos_len} + +{watchtime_status} + + Subscriptions: {subscriptions_len} + Cache usage: {cache_usage}" + ); + + Ok(()) +} + +pub fn config(app: &App) -> Result<()> { + let config_str = toml::to_string(&app.config)?; + + print!("{config_str}"); + + Ok(()) +} diff --git a/crates/yt/src/storage/migrate/mod.rs b/crates/yt/src/storage/migrate/mod.rs new file mode 100644 index 0000000..953d079 --- /dev/null +++ b/crates/yt/src/storage/migrate/mod.rs @@ -0,0 +1,279 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::{ + fmt::Display, + future::Future, + time::{SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context, Result, bail}; +use chrono::TimeDelta; +use log::{debug, info}; +use sqlx::{Sqlite, SqlitePool, Transaction, query}; + +use crate::app::App; + +macro_rules! make_upgrade { + ($app:expr, $old_version:expr, $new_version:expr, $sql_name:expr) => { + add_error_context( + async { + let mut tx = $app + .database + .begin() + .await + .context("Failed to start the update transaction")?; + debug!("Migrating: {} -> {}", $old_version, $new_version); + + sqlx::raw_sql(include_str!($sql_name)) + .execute(&mut *tx) + .await + .context("Failed to run the update sql script")?; + + set_db_version( + &mut tx, + if $old_version == Self::Empty { + // There is no previous version we would need to remove + None + } else { + Some($old_version) + }, + $new_version, + ) + .await + .with_context(|| format!("Failed to set the new version ({})", $new_version))?; + + tx.commit() + .await + .context("Failed to commit the update transaction")?; + + // NOTE: This is needed, so that sqlite "sees" our changes to the table + // without having to reconnect. <2025-02-18> + query!("VACUUM") + .execute(&$app.database) + .await + .context("Failed to vacuum database")?; + + Ok(()) + }, + $new_version, + ) + .await?; + + Box::pin($new_version.update($app)).await.context(concat!( + "While updating to version: ", + stringify!($new_version) + )) + }; +} + +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub enum DbVersion { + /// The database is not yet initialized. + Empty, + + /// The first database version. + /// Introduced: 2025-02-16. + Zero, + + /// Introduced: 2025-02-17. + One, + + /// Introduced: 2025-02-18. + Two, + + /// Introduced: 2025-03-21. + Three, +} +const CURRENT_VERSION: DbVersion = DbVersion::Three; + +async fn add_error_context( + function: impl Future>, + level: DbVersion, +) -> Result<()> { + function + .await + .with_context(|| format!("Failed to migrate database to version: {level}")) +} + +async fn set_db_version( + tx: &mut Transaction<'_, Sqlite>, + old_version: Option, + new_version: DbVersion, +) -> Result<()> { + let valid_from = get_current_date(); + + if let Some(old_version) = old_version { + let valid_to = valid_from + 1; + let old_version = old_version.as_sql_integer(); + + query!( + "UPDATE version SET valid_to = ? WHERE namespace = 'yt' AND number = ?;", + valid_to, + old_version + ) + .execute(&mut *(*tx)) + .await?; + } + + let version = new_version.as_sql_integer(); + + query!( + "INSERT INTO version (namespace, number, valid_from, valid_to) VALUES ('yt', ?, ?, NULL);", + version, + valid_from + ) + .execute(&mut *(*tx)) + .await?; + + Ok(()) +} + +impl DbVersion { + fn as_sql_integer(self) -> i32 { + match self { + DbVersion::Zero => 0, + DbVersion::One => 1, + DbVersion::Two => 2, + DbVersion::Three => 3, + + DbVersion::Empty => unreachable!("A empty version does not have an associated integer"), + } + } + + fn from_db(number: i64, namespace: &str) -> Result { + match (number, namespace) { + (0, "yt") => Ok(DbVersion::Zero), + (1, "yt") => Ok(DbVersion::One), + (2, "yt") => Ok(DbVersion::Two), + (3, "yt") => Ok(DbVersion::Three), + + (0, other) => bail!("Db version is Zero, but got unknown namespace: '{other}'"), + (1, other) => bail!("Db version is One, but got unknown namespace: '{other}'"), + (2, other) => bail!("Db version is Two, but got unknown namespace: '{other}'"), + (3, other) => bail!("Db version is Three, but got unknown namespace: '{other}'"), + + (other, "yt") => bail!("Got unkown version for 'yt' namespace: {other}"), + (num, nasp) => bail!("Got unkown version number ({num}) and namespace ('{nasp}')"), + } + } + + /// Try to update the database from version [`self`] to the [`CURRENT_VERSION`]. + /// + /// Each update is atomic, so if this function fails you are still guaranteed to have a + /// database at version `get_version`. + #[allow(clippy::too_many_lines)] + async fn update(self, app: &App) -> Result<()> { + match self { + Self::Empty => { + make_upgrade! {app, Self::Empty, Self::Zero, "./sql/0_Empty_to_Zero.sql"} + } + + Self::Zero => { + make_upgrade! {app, Self::Zero, Self::One, "./sql/1_Zero_to_One.sql"} + } + + Self::One => { + make_upgrade! {app, Self::One, Self::Two, "./sql/2_One_to_Two.sql"} + } + + Self::Two => { + make_upgrade! {app, Self::Two, Self::Three, "./sql/3_Two_to_Three.sql"} + } + + // This is the current_version + Self::Three => { + assert_eq!(self, CURRENT_VERSION); + assert_eq!(self, get_version(app).await?); + Ok(()) + } + } + } +} +impl Display for DbVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // It is a unit only enum, thus we can simply use the Debug formatting + ::fmt(self, f) + } +} + +/// Returns the current data as UNIX time stamp. +fn get_current_date() -> i64 { + let start = SystemTime::now(); + let seconds_since_epoch: TimeDelta = TimeDelta::from_std( + start + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"), + ) + .expect("Time does not go backwards"); + + // All database dates should be after the UNIX_EPOCH (and thus positiv) + seconds_since_epoch.num_milliseconds() +} + +/// Return the current database version. +/// +/// # Panics +/// Only if internal assertions fail. +pub async fn get_version(app: &App) -> Result { + get_version_db(&app.database).await +} +/// Return the current database version. +/// +/// In contrast to the [`get_version`] function, this function does not +/// a fully instantiated [`App`], a database connection suffices. +/// +/// # Panics +/// Only if internal assertions fail. +pub async fn get_version_db(pool: &SqlitePool) -> Result { + let version_table_exists = { + let query = query!( + "SELECT 1 as result FROM sqlite_master WHERE type = 'table' AND name = 'version'" + ) + .fetch_optional(pool) + .await?; + if let Some(output) = query { + assert_eq!(output.result, 1); + true + } else { + false + } + }; + if !version_table_exists { + return Ok(DbVersion::Empty); + } + + let current_version = query!( + " + SELECT namespace, number FROM version WHERE valid_to IS NULL; + " + ) + .fetch_one(pool) + .await + .context("Failed to fetch version number")?; + + DbVersion::from_db(current_version.number, current_version.namespace.as_str()) +} + +pub async fn migrate_db(app: &App) -> Result<()> { + let current_version = get_version(app) + .await + .context("Failed to determine initial version")?; + + if current_version == CURRENT_VERSION { + return Ok(()); + } + + info!("Migrate database from version '{current_version}' to version '{CURRENT_VERSION}'"); + + current_version.update(app).await?; + + Ok(()) +} diff --git a/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql new file mode 100644 index 0000000..d703bfc --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/0_Empty_to_Zero.sql @@ -0,0 +1,72 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2024 Benedikt Peetz +-- Copyright (C) 2025 Benedikt Peetz +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see . + +-- All tables should be declared STRICT, as I actually like to have types checking (and a +-- db that doesn't lie to me). + +-- Keep this table in sync with the `DbVersion` enumeration. +CREATE TABLE version ( + -- The `namespace` is only useful, if other tools ever build on this database + namespace TEXT NOT NULL, + + -- The version. + number INTEGER UNIQUE NOT NULL PRIMARY KEY, + + -- The validity of this version as UNIX time stamp + valid_from INTEGER NOT NULL CHECK (valid_from < valid_to), + -- If set to `NULL`, represents the current version + valid_to INTEGER UNIQUE CHECK (valid_to > valid_from) +) STRICT; + +-- Keep this table in sync with the `Video` structure +CREATE TABLE videos ( + cache_path TEXT UNIQUE CHECK (CASE WHEN cache_path IS NOT NULL THEN + status == 2 + ELSE + 1 + END), + description TEXT, + duration REAL, + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + last_status_change INTEGER NOT NULL, + parent_subscription_name TEXT, + priority INTEGER NOT NULL DEFAULT 0, + publish_date INTEGER, + status INTEGER NOT NULL DEFAULT 0 CHECK (status IN (0, 1, 2, 3, 4, 5) AND + CASE WHEN status == 2 THEN + cache_path IS NOT NULL + ELSE + 1 + END AND + CASE WHEN status != 2 THEN + cache_path IS NULL + ELSE + 1 + END), + status_change INTEGER NOT NULL DEFAULT 0 CHECK (status_change IN (0, 1)), + thumbnail_url TEXT, + title TEXT NOT NULL, + url TEXT UNIQUE NOT NULL +) STRICT; + +-- Store additional metadata for the videos marked to be watched +CREATE TABLE video_options ( + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + subtitle_langs TEXT NOT NULL, + playback_speed REAL NOT NULL, + FOREIGN KEY(extractor_hash) REFERENCES videos (extractor_hash) +) STRICT; + +-- Store subscriptions +CREATE TABLE subscriptions ( + name TEXT UNIQUE NOT NULL PRIMARY KEY, + url TEXT NOT NULL +) STRICT; diff --git a/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql new file mode 100644 index 0000000..da9315b --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/1_Zero_to_One.sql @@ -0,0 +1,28 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see . + +-- Is the video currently in a playlist? +ALTER TABLE videos ADD in_playlist INTEGER NOT NULL DEFAULT 0 CHECK (in_playlist IN (0, 1)); +UPDATE videos SET in_playlist = 0; + +-- Is it 'focused' (i.e., the select video)? +-- Only of video should be focused at a time. +ALTER TABLE videos +ADD COLUMN is_focused INTEGER NOT NULL DEFAULT 0 +CHECK (is_focused IN (0, 1)); +UPDATE videos SET is_focused = 0; + +-- The progress the user made in watching the video. +ALTER TABLE videos ADD watch_progress INTEGER NOT NULL DEFAULT 0 CHECK (watch_progress <= duration); +-- Assume, that the user has watched the video to end, if it is marked as watched +UPDATE videos SET watch_progress = ifnull(duration, 0) WHERE status = 3; +UPDATE videos SET watch_progress = 0 WHERE status != 3; + +ALTER TABLE videos DROP COLUMN status_change; diff --git a/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql new file mode 100644 index 0000000..806de07 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/2_One_to_Two.sql @@ -0,0 +1,11 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see . + +ALTER TABLE videos DROP in_playlist; diff --git a/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql new file mode 100644 index 0000000..b33f849 --- /dev/null +++ b/crates/yt/src/storage/migrate/sql/3_Two_to_Three.sql @@ -0,0 +1,85 @@ +-- yt - A fully featured command line YouTube client +-- +-- Copyright (C) 2025 Benedikt Peetz +-- SPDX-License-Identifier: GPL-3.0-or-later +-- +-- This file is part of Yt. +-- +-- You should have received a copy of the License along with this program. +-- If not, see . + + +-- 1. Create new table +-- 2. Copy data +-- 3. Drop old table +-- 4. Rename new into old + +-- remove the original TRANSACTION +COMMIT TRANSACTION; + +-- tweak config +PRAGMA foreign_keys=OFF; + +-- start your own TRANSACTION +BEGIN TRANSACTION; + +CREATE TABLE videos_new ( + cache_path TEXT UNIQUE CHECK (CASE + WHEN cache_path IS NOT NULL THEN status == 2 + ELSE 1 + END), + description TEXT, + duration REAL, + extractor_hash TEXT UNIQUE NOT NULL PRIMARY KEY, + last_status_change INTEGER NOT NULL, + parent_subscription_name TEXT, + priority INTEGER NOT NULL DEFAULT 0, + publish_date INTEGER, + status INTEGER NOT NULL DEFAULT 0 CHECK (status IN (0, 1, 2, 3, 4, 5) AND + CASE + WHEN status == 2 THEN cache_path IS NOT NULL + WHEN status != 2 THEN cache_path IS NULL + ELSE 1 + END), + thumbnail_url TEXT, + title TEXT NOT NULL, + url TEXT UNIQUE NOT NULL, + is_focused INTEGER UNIQUE DEFAULT NULL CHECK (CASE + WHEN is_focused IS NOT NULL THEN is_focused == 1 + ELSE 1 + END), + watch_progress INTEGER NOT NULL DEFAULT 0 CHECK (watch_progress <= duration) +) STRICT; + +INSERT INTO videos_new SELECT + videos.cache_path, + videos.description, + videos.duration, + videos.extractor_hash, + videos.last_status_change, + videos.parent_subscription_name, + videos.priority, + videos.publish_date, + videos.status, + videos.thumbnail_url, + videos.title, + videos.url, + dummy.is_focused, + videos.watch_progress +FROM videos, (SELECT NULL AS is_focused) AS dummy; + +DROP TABLE videos; + +ALTER TABLE videos_new RENAME TO videos; + +-- check foreign key constraint still upholding. +PRAGMA foreign_key_check; + +-- commit your own TRANSACTION +COMMIT TRANSACTION; + +-- rollback all config you setup before. +PRAGMA foreign_keys=ON; + +-- start a new TRANSACTION to let migrator commit it. +BEGIN TRANSACTION; diff --git a/crates/yt/src/storage/mod.rs b/crates/yt/src/storage/mod.rs new file mode 100644 index 0000000..d352b41 --- /dev/null +++ b/crates/yt/src/storage/mod.rs @@ -0,0 +1,14 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +pub mod migrate; +pub mod subscriptions; +pub mod video_database; diff --git a/crates/yt/src/storage/subscriptions.rs b/crates/yt/src/storage/subscriptions.rs new file mode 100644 index 0000000..6c0d08a --- /dev/null +++ b/crates/yt/src/storage/subscriptions.rs @@ -0,0 +1,141 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +//! Handle subscriptions + +use std::collections::HashMap; + +use anyhow::Result; +use log::debug; +use sqlx::query; +use url::Url; +use yt_dlp::YoutubeDLOptions; + +use crate::{app::App, unreachable::Unreachable}; + +#[derive(Clone, Debug)] +pub struct Subscription { + /// The human readable name of this subscription + pub name: String, + + /// The URL this subscription subscribes to + pub url: Url, +} + +impl Subscription { + #[must_use] + pub fn new(name: String, url: Url) -> Self { + Self { name, url } + } +} + +/// Check whether an URL could be used as a subscription URL +pub async fn check_url(url: Url) -> Result { + let yt_dlp = YoutubeDLOptions::new() + .set("playliststart", 1) + .set("playlistend", 10) + .set("noplaylist", false) + .set("extract_flat", "in_playlist") + .build()?; + + let info = yt_dlp.extract_info(&url, false, false)?; + + debug!("{:#?}", info); + + Ok(info.get("_type") == Some(&serde_json::Value::String("Playlist".to_owned()))) +} + +#[derive(Default, Debug)] +pub struct Subscriptions(pub(crate) HashMap); + +/// Remove all subscriptions +pub async fn remove_all(app: &App) -> Result<()> { + query!( + " + DELETE FROM subscriptions; + ", + ) + .execute(&app.database) + .await?; + + Ok(()) +} + +/// Get a list of subscriptions +pub async fn get(app: &App) -> Result { + let raw_subs = query!( + " + SELECT * + FROM subscriptions; + " + ) + .fetch_all(&app.database) + .await?; + + let subscriptions: HashMap = raw_subs + .into_iter() + .map(|sub| { + ( + sub.name.clone(), + Subscription::new( + sub.name, + Url::parse(&sub.url).unreachable("It was an URL, when we inserted it."), + ), + ) + }) + .collect(); + + Ok(Subscriptions(subscriptions)) +} + +pub async fn add_subscription(app: &App, sub: &Subscription) -> Result<()> { + let url = sub.url.to_string(); + + query!( + " + INSERT INTO subscriptions ( + name, + url + ) VALUES (?, ?); + ", + sub.name, + url + ) + .execute(&app.database) + .await?; + + println!("Subscribed to '{}' at '{}'", sub.name, sub.url); + Ok(()) +} + +/// # Panics +/// Only if assertions fail +pub async fn remove_subscription(app: &App, sub: &Subscription) -> Result<()> { + let output = query!( + " + DELETE FROM subscriptions + WHERE name = ? + ", + sub.name, + ) + .execute(&app.database) + .await?; + + assert_eq!( + output.rows_affected(), + 1, + "The remove subscriptino query did effect more (or less) than one row. This is a bug." + ); + + println!("Unsubscribed from '{}' at '{}'", sub.name, sub.url); + + Ok(()) +} diff --git a/crates/yt/src/storage/video_database/downloader.rs b/crates/yt/src/storage/video_database/downloader.rs new file mode 100644 index 0000000..a95081e --- /dev/null +++ b/crates/yt/src/storage/video_database/downloader.rs @@ -0,0 +1,130 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use log::debug; +use sqlx::query; + +use crate::{ + app::App, + storage::video_database::{VideoStatus, VideoStatusMarker}, + unreachable::Unreachable, + video_from_record, +}; + +use super::{ExtractorHash, Video}; + +/// Returns to next video which should be downloaded. This respects the priority assigned by select. +/// It does not return videos, which are already cached. +/// +/// # Panics +/// Only if assertions fail. +pub async fn get_next_uncached_video(app: &App) -> Result> { + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + // NOTE: The ORDER BY statement should be the same as the one in [`get::videos`].<2024-08-22> + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND cache_path IS NULL + ORDER BY priority DESC, publish_date DESC + LIMIT 1; + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + Ok(Some(video_from_record! {base})) + } +} + +/// Update the cached path of a video. Will be set to NULL if the path is None +/// This will also set the status to `Cached` when path is Some, otherwise it set's the status to +/// `Watch`. +pub async fn set_video_cache_path( + app: &App, + video: &ExtractorHash, + path: Option<&Path>, +) -> Result<()> { + if let Some(path) = path { + debug!( + "Setting cache path from '{}' to '{}'", + video.into_short_hash(app).await?, + path.display() + ); + + let path_str = path.display().to_string(); + let extractor_hash = video.hash().to_string(); + let status = VideoStatusMarker::Cached.as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = ?, status = ? + WHERE extractor_hash = ?; + "#, + path_str, + status, + extractor_hash + ) + .execute(&app.database) + .await?; + + Ok(()) + } else { + debug!( + "Setting cache path from '{}' to NULL", + video.into_short_hash(app).await?, + ); + + let extractor_hash = video.hash().to_string(); + let status = VideoStatus::Watch.as_marker().as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = NULL, status = ? + WHERE extractor_hash = ?; + "#, + status, + extractor_hash + ) + .execute(&app.database) + .await?; + + Ok(()) + } +} + +/// Returns the number of cached videos +pub async fn get_allocated_cache(app: &App) -> Result { + let count = query!( + r#" + SELECT COUNT(cache_path) as count + FROM videos + WHERE cache_path IS NOT NULL; +"#, + ) + .fetch_one(&app.database) + .await?; + + Ok(u32::try_from(count.count) + .unreachable("The value should be strictly positive (and bolow `u32::Max`)")) +} diff --git a/crates/yt/src/storage/video_database/extractor_hash.rs b/crates/yt/src/storage/video_database/extractor_hash.rs new file mode 100644 index 0000000..df545d7 --- /dev/null +++ b/crates/yt/src/storage/video_database/extractor_hash.rs @@ -0,0 +1,163 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::{collections::HashSet, fmt::Display, str::FromStr}; + +use anyhow::{Context, Result, bail}; +use blake3::Hash; +use log::debug; +use tokio::sync::OnceCell; + +use crate::{app::App, storage::video_database::get::get_all_hashes, unreachable::Unreachable}; + +static EXTRACTOR_HASH_LENGTH: OnceCell = OnceCell::const_new(); + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Copy)] +pub struct ExtractorHash { + hash: Hash, +} + +impl Display for ExtractorHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.hash.fmt(f) + } +} + +#[derive(Debug, Clone)] +pub struct ShortHash(String); + +impl Display for ShortHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone)] +#[allow(clippy::module_name_repetitions)] +pub struct LazyExtractorHash { + value: ShortHash, +} + +impl FromStr for LazyExtractorHash { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + // perform some cheap validation + if s.len() > 64 { + bail!("A hash can only contain 64 bytes!"); + } + + Ok(Self { + value: ShortHash(s.to_owned()), + }) + } +} + +impl LazyExtractorHash { + /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] + pub async fn realize(self, app: &App) -> Result { + ExtractorHash::from_short_hash(app, &self.value).await + } +} + +impl ExtractorHash { + #[must_use] + pub fn from_hash(hash: Hash) -> Self { + Self { hash } + } + pub async fn from_short_hash(app: &App, s: &ShortHash) -> Result { + Ok(Self { + hash: Self::short_hash_to_full_hash(app, s).await?, + }) + } + + #[must_use] + pub fn hash(&self) -> &Hash { + &self.hash + } + + pub async fn into_short_hash(&self, app: &App) -> Result { + let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { + *needed_chars + } else { + let needed_chars = self + .get_needed_char_len(app) + .await + .context("Failed to calculate needed char length")?; + EXTRACTOR_HASH_LENGTH.set(needed_chars).unreachable( + "This should work at this stage, as we checked above that it is empty.", + ); + + needed_chars + }; + + Ok(ShortHash( + self.hash() + .to_hex() + .chars() + .take(needed_chars) + .collect::(), + )) + } + + async fn short_hash_to_full_hash(app: &App, s: &ShortHash) -> Result { + let all_hashes = get_all_hashes(app) + .await + .context("Failed to fetch all extractor -hashesh from database")?; + + let needed_chars = s.0.len(); + + for hash in all_hashes { + if hash.to_hex()[..needed_chars] == s.0 { + return Ok(hash); + } + } + + bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); + } + + async fn get_needed_char_len(&self, app: &App) -> Result { + debug!("Calculating the needed hash char length"); + let all_hashes = get_all_hashes(app) + .await + .context("Failed to fetch all extractor -hashesh from database")?; + + let all_char_vec_hashes = all_hashes + .into_iter() + .map(|hash| hash.to_hex().chars().collect::>()) + .collect::>>(); + + // This value should be updated later, if not rust will panic in the assertion. + let mut needed_chars: usize = 1000; + 'outer: for i in 1..64 { + let i_chars: Vec = all_char_vec_hashes + .iter() + .map(|vec| vec.iter().take(i).collect::()) + .collect(); + + let mut uniqnes_hashmap: HashSet = HashSet::new(); + for ch in i_chars { + if !uniqnes_hashmap.insert(ch) { + // The key was already in the hash map, thus we have a duplicated char and need + // at least one char more + continue 'outer; + } + } + + needed_chars = i; + break 'outer; + } + + assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); + + Ok(needed_chars) + } +} diff --git a/crates/yt/src/storage/video_database/get/mod.rs b/crates/yt/src/storage/video_database/get/mod.rs new file mode 100644 index 0000000..0456cd3 --- /dev/null +++ b/crates/yt/src/storage/video_database/get/mod.rs @@ -0,0 +1,307 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// Copyright (C) 2025 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +//! These functions interact with the storage db in a read-only way. They are added on-demand (as +//! you could theoretically just could do everything with the `get_videos` function), as +//! performance or convince requires. +use std::{fs::File, path::PathBuf}; + +use anyhow::{Context, Result, bail}; +use blake3::Hash; +use log::{debug, trace}; +use sqlx::query; +use yt_dlp::InfoJson; + +use crate::{ + app::App, + storage::{ + subscriptions::Subscription, + video_database::{Video, extractor_hash::ExtractorHash}, + }, + unreachable::Unreachable, +}; + +use super::{MpvOptions, VideoOptions, VideoStatus, VideoStatusMarker, YtDlpOptions}; + +mod playlist; +pub use playlist::*; + +#[macro_export] +macro_rules! video_from_record { + ($record:expr) => { + Video { + description: $record.description.clone(), + duration: $crate::storage::video_database::MaybeDuration::from_maybe_secs_f64( + $record.duration, + ), + extractor_hash: + $crate::storage::video_database::extractor_hash::ExtractorHash::from_hash( + $record + .extractor_hash + .parse() + .expect("The db hash should be a valid blake3 hash"), + ), + last_status_change: $crate::storage::video_database::TimeStamp::from_secs( + $record.last_status_change, + ), + parent_subscription_name: $record.parent_subscription_name.clone(), + publish_date: $record + .publish_date + .map(|pd| $crate::storage::video_database::TimeStamp::from_secs(pd)), + status: { + let marker = $crate::storage::video_database::VideoStatusMarker::from_db_integer( + $record.status, + ); + + let optional = if let Some(cache_path) = &$record.cache_path { + Some(( + PathBuf::from(cache_path), + if $record.is_focused == Some(1) { + true + } else { + false + }, + )) + } else { + None + }; + + $crate::storage::video_database::VideoStatus::from_marker(marker, optional) + }, + thumbnail_url: if let Some(url) = &$record.thumbnail_url { + Some(url::Url::parse(&url).expect("Parsing this as url should always work")) + } else { + None + }, + title: $record.title.clone(), + url: url::Url::parse(&$record.url).expect("Parsing this as url should always work"), + priority: $crate::storage::video_database::Priority::from($record.priority), + + watch_progress: std::time::Duration::from_secs( + u64::try_from($record.watch_progress).expect("The record is positive i64"), + ), + } + }; +} + +/// Returns the videos that are in the `allowed_states`. +/// +/// # Panics +/// Only, if assertions fail. +pub async fn videos(app: &App, allowed_states: &[VideoStatusMarker]) -> Result> { + fn test(all_states: &[VideoStatusMarker], check: VideoStatusMarker) -> Option { + if all_states.contains(&check) { + trace!("State '{check:?}' marked as active"); + Some(check.as_db_integer()) + } else { + trace!("State '{check:?}' marked as inactive"); + None + } + } + fn states_to_string(allowed_states: &[VideoStatusMarker]) -> String { + let mut states = allowed_states + .iter() + .fold(String::from("&["), |mut acc, state| { + acc.push_str(state.as_str()); + acc.push_str(", "); + acc + }); + states = states.trim().to_owned(); + states = states.trim_end_matches(',').to_owned(); + states.push(']'); + states + } + + debug!( + "Fetching videos in the states: '{}'", + states_to_string(allowed_states) + ); + let active_pick: Option = test(allowed_states, VideoStatusMarker::Pick); + let active_watch: Option = test(allowed_states, VideoStatusMarker::Watch); + let active_cached: Option = test(allowed_states, VideoStatusMarker::Cached); + let active_watched: Option = test(allowed_states, VideoStatusMarker::Watched); + let active_drop: Option = test(allowed_states, VideoStatusMarker::Drop); + let active_dropped: Option = test(allowed_states, VideoStatusMarker::Dropped); + + let videos = query!( + r" + SELECT * + FROM videos + WHERE status IN (?,?,?,?,?,?) + ORDER BY priority DESC, publish_date DESC; + ", + active_pick, + active_watch, + active_cached, + active_watched, + active_drop, + active_dropped, + ) + .fetch_all(&app.database) + .await + .with_context(|| { + format!( + "Failed to query videos with states: '{}'", + states_to_string(allowed_states) + ) + })?; + + let real_videos: Vec