From 6c9286857ef8b314962b67f4a16a66e8c35531bc Mon Sep 17 00:00:00 2001 From: Benedikt Peetz Date: Mon, 14 Oct 2024 14:56:29 +0200 Subject: refactor(treewide): Combine the separate crates in one workspace --- src/storage/mod.rs | 12 - src/storage/subscriptions.rs | 140 ----------- src/storage/video_database/downloader.rs | 153 ------------ src/storage/video_database/extractor_hash.rs | 159 ------------ src/storage/video_database/getters.rs | 345 --------------------------- src/storage/video_database/mod.rs | 179 -------------- src/storage/video_database/schema.sql | 57 ----- src/storage/video_database/setters.rs | 270 --------------------- 8 files changed, 1315 deletions(-) delete mode 100644 src/storage/mod.rs delete mode 100644 src/storage/subscriptions.rs delete mode 100644 src/storage/video_database/downloader.rs delete mode 100644 src/storage/video_database/extractor_hash.rs delete mode 100644 src/storage/video_database/getters.rs delete mode 100644 src/storage/video_database/mod.rs delete mode 100644 src/storage/video_database/schema.sql delete mode 100644 src/storage/video_database/setters.rs (limited to 'src/storage') diff --git a/src/storage/mod.rs b/src/storage/mod.rs deleted file mode 100644 index 6a12d8b..0000000 --- a/src/storage/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see . - -pub mod subscriptions; -pub mod video_database; diff --git a/src/storage/subscriptions.rs b/src/storage/subscriptions.rs deleted file mode 100644 index 22edd08..0000000 --- a/src/storage/subscriptions.rs +++ /dev/null @@ -1,140 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see . - -//! Handle subscriptions - -use std::collections::HashMap; - -use anyhow::Result; -use log::debug; -use serde_json::{json, Value}; -use sqlx::query; -use url::Url; -use yt_dlp::wrapper::info_json::InfoType; - -use crate::app::App; - -#[derive(Clone, Debug)] -pub struct Subscription { - /// The human readable name of this subscription - pub name: String, - - /// The URL this subscription subscribes to - pub url: Url, -} - -impl Subscription { - pub fn new(name: String, url: Url) -> Self { - Self { name, url } - } -} - -/// Check whether an URL could be used as a subscription URL -pub async fn check_url(url: &Url) -> Result { - let yt_opts = match json!( { - "playliststart": 1, - "playlistend": 10, - "noplaylist": false, - "extract_flat": "in_playlist", - }) { - Value::Object(map) => map, - _ => unreachable!("This is hardcoded"), - }; - - let info = yt_dlp::extract_info(&yt_opts, url, false, false).await?; - - debug!("{:#?}", info); - - Ok(info._type == Some(InfoType::Playlist)) -} - -#[derive(Default)] -pub struct Subscriptions(pub(crate) HashMap); - -pub async fn remove_all_subscriptions(app: &App) -> Result<()> { - query!( - " - DELETE FROM subscriptions; - ", - ) - .execute(&app.database) - .await?; - - Ok(()) -} - -/// Get a list of subscriptions -pub async fn get_subscriptions(app: &App) -> Result { - let raw_subs = query!( - " - SELECT * - FROM subscriptions; - " - ) - .fetch_all(&app.database) - .await?; - - let subscriptions: HashMap = raw_subs - .into_iter() - .map(|sub| { - ( - sub.name.clone(), - Subscription::new( - sub.name, - Url::parse(&sub.url).expect("This should be valid"), - ), - ) - }) - .collect(); - - Ok(Subscriptions(subscriptions)) -} - -pub async fn add_subscription(app: &App, sub: &Subscription) -> Result<()> { - let url = sub.url.to_string(); - - query!( - " - INSERT INTO subscriptions ( - name, - url - ) VALUES (?, ?); - ", - sub.name, - url - ) - .execute(&app.database) - .await?; - - println!("Subscribed to '{}' at '{}'", sub.name, sub.url); - Ok(()) -} - -pub async fn remove_subscription(app: &App, sub: &Subscription) -> Result<()> { - let output = query!( - " - DELETE FROM subscriptions - WHERE name = ? - ", - sub.name, - ) - .execute(&app.database) - .await?; - - assert_eq!( - output.rows_affected(), - 1, - "The remove subscriptino query did effect more (or less) than one row. This is a bug." - ); - - println!("Unsubscribed from '{}' at '{}'", sub.name, sub.url); - - Ok(()) -} diff --git a/src/storage/video_database/downloader.rs b/src/storage/video_database/downloader.rs deleted file mode 100644 index ccd4ca9..0000000 --- a/src/storage/video_database/downloader.rs +++ /dev/null @@ -1,153 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see . - -use std::path::{Path, PathBuf}; - -use anyhow::Result; -use log::debug; -use sqlx::query; -use url::Url; - -use crate::{app::App, storage::video_database::VideoStatus}; - -use super::{ExtractorHash, Video}; - -/// Returns to next video which should be downloaded. This respects the priority assigned by select. -/// It does not return videos, which are already cached. -pub async fn get_next_uncached_video(app: &App) -> Result> { - let status = VideoStatus::Watch.as_db_integer(); - - // NOTE: The ORDER BY statement should be the same as the one in [`getters::get_videos`].<2024-08-22> - let result = query!( - r#" - SELECT * - FROM videos - WHERE status = ? AND cache_path IS NULL - ORDER BY priority DESC, publish_date DESC - LIMIT 1; - "#, - status - ) - .fetch_one(&app.database) - .await; - - if let Err(sqlx::Error::RowNotFound) = result { - Ok(None) - } else { - let base = result?; - - let thumbnail_url = base - .thumbnail_url - .as_ref() - .map(|url| Url::parse(url).expect("Parsing this as url should always work")); - - let status_change = if base.status_change == 1 { - true - } else { - assert_eq!(base.status_change, 0, "Can only be 1 or 0"); - false - }; - - let video = Video { - cache_path: base.cache_path.as_ref().map(PathBuf::from), - description: base.description.clone(), - duration: base.duration, - extractor_hash: ExtractorHash::from_hash( - base.extractor_hash - .parse() - .expect("The hash in the db should be valid"), - ), - last_status_change: base.last_status_change, - parent_subscription_name: base.parent_subscription_name.clone(), - priority: base.priority, - publish_date: base.publish_date, - status: VideoStatus::from_db_integer(base.status), - status_change, - thumbnail_url, - title: base.title.clone(), - url: Url::parse(&base.url).expect("Parsing this as url should always work"), - }; - - Ok(Some(video)) - } -} - -/// Update the cached path of a video. Will be set to NULL if the path is None -/// This will also set the status to `Cached` when path is Some, otherwise it set's the status to -/// `Watch`. -pub async fn set_video_cache_path( - app: &App, - video: &ExtractorHash, - path: Option<&Path>, -) -> Result<()> { - if let Some(path) = path { - debug!( - "Setting cache path from '{}' to '{}'", - video.into_short_hash(app).await?, - path.display() - ); - - let path_str = path.display().to_string(); - let extractor_hash = video.hash().to_string(); - let status = VideoStatus::Cached.as_db_integer(); - - query!( - r#" - UPDATE videos - SET cache_path = ?, status = ? - WHERE extractor_hash = ?; - "#, - path_str, - status, - extractor_hash - ) - .execute(&app.database) - .await?; - - Ok(()) - } else { - debug!( - "Setting cache path from '{}' to NULL", - video.into_short_hash(app).await?, - ); - - let extractor_hash = video.hash().to_string(); - let status = VideoStatus::Watch.as_db_integer(); - - query!( - r#" - UPDATE videos - SET cache_path = NULL, status = ? - WHERE extractor_hash = ?; - "#, - status, - extractor_hash - ) - .execute(&app.database) - .await?; - - Ok(()) - } -} - -/// Returns the number of cached videos -pub async fn get_allocated_cache(app: &App) -> Result { - let count = query!( - r#" - SELECT COUNT(cache_path) as count - FROM videos - WHERE cache_path IS NOT NULL; -"#, - ) - .fetch_one(&app.database) - .await?; - - Ok(count.count as u32) -} diff --git a/src/storage/video_database/extractor_hash.rs b/src/storage/video_database/extractor_hash.rs deleted file mode 100644 index c956919..0000000 --- a/src/storage/video_database/extractor_hash.rs +++ /dev/null @@ -1,159 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see . - -use std::{collections::HashMap, fmt::Display, str::FromStr}; - -use anyhow::{bail, Context, Result}; -use blake3::Hash; -use log::debug; -use tokio::sync::OnceCell; - -use crate::{app::App, storage::video_database::getters::get_all_hashes}; - -static EXTRACTOR_HASH_LENGTH: OnceCell = OnceCell::const_new(); - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ExtractorHash { - hash: Hash, -} - -impl Display for ExtractorHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.hash.fmt(f) - } -} - -#[derive(Debug, Clone)] -pub struct ShortHash(String); - -impl Display for ShortHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -#[derive(Debug, Clone)] -pub struct LazyExtractorHash { - value: ShortHash, -} - -impl FromStr for LazyExtractorHash { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::result::Result { - // perform some cheap validation - if s.len() > 64 { - bail!("A hash can only contain 64 bytes!"); - } - - Ok(Self { - value: ShortHash(s.to_owned()), - }) - } -} - -impl LazyExtractorHash { - /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] - pub async fn realize(self, app: &App) -> Result { - ExtractorHash::from_short_hash(app, &self.value).await - } -} - -impl ExtractorHash { - pub fn from_hash(hash: Hash) -> Self { - Self { hash } - } - pub async fn from_short_hash(app: &App, s: &ShortHash) -> Result { - Ok(Self { - hash: Self::short_hash_to_full_hash(app, s).await?, - }) - } - - pub fn hash(&self) -> &Hash { - &self.hash - } - - pub async fn into_short_hash(&self, app: &App) -> Result { - let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { - *needed_chars - } else { - let needed_chars = self - .get_needed_char_len(app) - .await - .context("Failed to calculate needed char length")?; - EXTRACTOR_HASH_LENGTH - .set(needed_chars) - .expect("This should work at this stage"); - - needed_chars - }; - - Ok(ShortHash( - self.hash() - .to_hex() - .chars() - .take(needed_chars) - .collect::(), - )) - } - - async fn short_hash_to_full_hash(app: &App, s: &ShortHash) -> Result { - let all_hashes = get_all_hashes(app) - .await - .context("Failed to fetch all extractor -hashesh from database")?; - - let needed_chars = s.0.len(); - - for hash in all_hashes { - if hash.to_hex()[..needed_chars] == s.0 { - return Ok(hash); - } - } - - bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); - } - - async fn get_needed_char_len(&self, app: &App) -> Result { - debug!("Calculating the needed hash char length"); - let all_hashes = get_all_hashes(app) - .await - .context("Failed to fetch all extractor -hashesh from database")?; - - let all_char_vec_hashes = all_hashes - .into_iter() - .map(|hash| hash.to_hex().chars().collect::>()) - .collect::>>(); - - // This value should be updated later, if not rust will panic in the assertion. - let mut needed_chars: usize = 1000; - 'outer: for i in 1..64 { - let i_chars: Vec = all_char_vec_hashes - .iter() - .map(|vec| vec.iter().take(i).collect::()) - .collect(); - - let mut uniqnes_hashmap: HashMap = HashMap::new(); - for ch in i_chars { - if let Some(()) = uniqnes_hashmap.insert(ch, ()) { - // The key was already in the hash map, thus we have a duplicated char and need - // at least one char more - continue 'outer; - } - } - - needed_chars = i; - break 'outer; - } - - assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); - - Ok(needed_chars) - } -} diff --git a/src/storage/video_database/getters.rs b/src/storage/video_database/getters.rs deleted file mode 100644 index 29dd014..0000000 --- a/src/storage/video_database/getters.rs +++ /dev/null @@ -1,345 +0,0 @@ -// yt - A fully featured command line YouTube client -// -// Copyright (C) 2024 Benedikt Peetz -// SPDX-License-Identifier: GPL-3.0-or-later -// -// This file is part of Yt. -// -// You should have received a copy of the License along with this program. -// If not, see . - -//! These functions interact with the storage db in a read-only way. They are added on-demaned (as -//! you could theoretically just could do everything with the `get_videos` function), as -//! performance or convince requires. -use std::{fs::File, path::PathBuf}; - -use anyhow::{bail, Context, Result}; -use blake3::Hash; -use log::debug; -use sqlx::{query, QueryBuilder, Row, Sqlite}; -use url::Url; -use yt_dlp::wrapper::info_json::InfoJson; - -use crate::{ - app::App, - storage::{ - subscriptions::Subscription, - video_database::{extractor_hash::ExtractorHash, Video}, - }, -}; - -use super::{MpvOptions, VideoOptions, VideoStatus, YtDlpOptions}; - -macro_rules! video_from_record { - ($record:expr) => { - let thumbnail_url = if let Some(url) = &$record.thumbnail_url { - Some(Url::parse(&url).expect("Parsing this as url should always work")) - } else { - None - }; - - Ok(Video { - cache_path: $record.cache_path.as_ref().map(|val| PathBuf::from(val)), - description: $record.description.clone(), - duration: $record.duration, - extractor_hash: ExtractorHash::from_hash( - $record - .extractor_hash - .parse() - .expect("The db hash should be a valid blake3 hash"), - ), - last_status_change: $record.last_status_change, - parent_subscription_name: $record.parent_subscription_name.clone(), - publish_date: $record.publish_date, - status: VideoStatus::from_db_integer($record.status), - thumbnail_url, - title: $record.title.clone(), - url: Url::parse(&$record.url).expect("Parsing this as url should always work"), - priority: $record.priority, - status_change: if $record.status_change == 1 { - true - } else { - assert_eq!($record.status_change, 0); - false - }, - }) - }; -} - -/// Get the lines to display at the selection file -/// [`changing` = true]: Means that we include *only* videos, that have the `status_changing` flag set -/// [`changing` = None]: Means that we include *both* videos, that have the `status_changing` flag set and not set -pub async fn get_videos( - app: &App, - allowed_states: &[VideoStatus], - changing: Option, -) -> Result> { - let mut qb: QueryBuilder = QueryBuilder::new( - "\ - SELECT * - FROM videos - WHERE status IN ", - ); - - qb.push("("); - allowed_states - .iter() - .enumerate() - .for_each(|(index, state)| { - qb.push("'"); - qb.push(state.as_db_integer()); - qb.push("'"); - - if index != allowed_states.len() - 1 { - qb.push(","); - } - }); - qb.push(")"); - - if let Some(val) = changing { - if val { - qb.push(" AND status_change = 1"); - } else { - qb.push(" AND status_change = 0"); - } - } - - qb.push("\n ORDER BY priority DESC, publish_date DESC;"); - - debug!("Will run: \"{}\"", qb.sql()); - - let videos = qb.build().fetch_all(&app.database).await.with_context(|| { - format!( - "Failed to query videos with states: '{}'", - allowed_states.iter().fold(String::new(), |mut acc, state| { - acc.push(' '); - acc.push_str(state.as_str()); - acc - }), - ) - })?; - - let real_videos: Vec