diff options
Diffstat (limited to 'src/update')
| -rw-r--r-- | src/update/mod.rs | 207 |
1 files changed, 207 insertions, 0 deletions
diff --git a/src/update/mod.rs b/src/update/mod.rs new file mode 100644 index 0000000..9128bf7 --- /dev/null +++ b/src/update/mod.rs @@ -0,0 +1,207 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. + +use std::{collections::HashMap, process::Stdio, str::FromStr}; + +use anyhow::{Context, Ok, Result}; +use chrono::{DateTime, Utc}; +use log::{error, info, warn}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::Command, +}; +use url::Url; +use yt_dlp::{unsmuggle_url, wrapper::info_json::InfoJson}; + +use crate::{ + app::App, + storage::{ + subscriptions::{get_subscriptions, Subscription}, + video_database::{ + extractor_hash::ExtractorHash, getters::get_all_hashes, setters::add_video, Video, + VideoStatus, + }, + }, +}; + +pub async fn update( + app: &App, + max_backlog: u32, + subs_to_update: Vec<String>, + _concurrent_processes: usize, +) -> Result<()> { + let subscriptions = get_subscriptions(&app).await?; + let mut back_subs: HashMap<Url, Subscription> = HashMap::new(); + + let mut urls: Vec<String> = vec![]; + for (name, sub) in subscriptions.0 { + if subs_to_update.contains(&name) || subs_to_update.is_empty() { + urls.push(sub.url.to_string()); + back_subs.insert(sub.url.clone(), sub); + } else { + info!( + "Not updating subscription '{}' as it was not specified", + name + ); + } + } + + let mut child = Command::new("./python_update/raw_update.py") + .arg(max_backlog.to_string()) + .args(&urls) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .stdin(Stdio::null()) + .spawn() + .context("Failed to call python3 update_raw")?; + + let mut out = BufReader::new( + child + .stdout + .take() + .expect("Should be able to take child stdout"), + ) + .lines(); + + let hashes = get_all_hashes(app).await?; + + while let Some(line) = out.next_line().await? { + // use tokio::{fs::File, io::AsyncWriteExt}; + // let mut output = File::create("output.json").await?; + // output.write(line.as_bytes()).await?; + // output.flush().await?; + // output.sync_all().await?; + // drop(output); + + let output_json: HashMap<Url, InfoJson> = + serde_json::from_str(&line).expect("This should be valid json"); + + for (url, value) in output_json { + let sub = back_subs.get(&url).expect("This was stored before"); + process_subscription(app, sub, value, &hashes).await? + } + } + + let out = child.wait().await?; + if out.success() { + error!("A yt update-once invokation failed for all subscriptions.") + } + + Ok(()) +} + +async fn process_subscription( + app: &App, + sub: &Subscription, + entry: InfoJson, + hashes: &Vec<blake3::Hash>, +) -> Result<()> { + macro_rules! unwrap_option { + ($option:expr) => { + match $option { + Some(x) => x, + None => anyhow::bail!(concat!( + "Expected a value, but '", + stringify!($option), + "' is None!" + )), + } + }; + } + + let publish_date = if let Some(date) = &entry.upload_date { + let year: u32 = date + .chars() + .take(4) + .collect::<String>() + .parse() + .expect("Should work."); + let month: u32 = date + .chars() + .skip(4) + .take(2) + .collect::<String>() + .parse() + .expect("Should work"); + let day: u32 = date + .chars() + .skip(6) + .take(2) + .collect::<String>() + .parse() + .expect("Should work"); + + let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z"); + Some( + DateTime::<Utc>::from_str(&date_string) + .expect("This should always work") + .timestamp(), + ) + } else { + warn!( + "The video '{}' lacks it's upload date!", + unwrap_option!(&entry.title) + ); + None + }; + + let thumbnail_url = match (&entry.thumbnails, &entry.thumbnail) { + (None, None) => None, + (None, Some(thumbnail)) => Some(thumbnail.to_owned()), + + // TODO: The algorithm is not exactly the best <2024-05-28> + (Some(thumbnails), None) => Some( + thumbnails + .get(0) + .expect("At least one should exist") + .url + .clone(), + ), + (Some(_), Some(thumnail)) => Some(thumnail.to_owned()), + }; + + let url = { + let smug_url: url::Url = unwrap_option!(entry.webpage_url.clone()); + unsmuggle_url(smug_url)? + }; + + let extractor_hash = blake3::hash(url.as_str().as_bytes()); + + if hashes.contains(&extractor_hash) { + // We already stored the video information + println!( + "(Ignoring duplicated video from: '{}' -> '{}')", + sub.name, + unwrap_option!(entry.title) + ); + return Ok(()); + } else { + let video = Video { + cache_path: None, + description: entry.description.clone(), + duration: entry.duration, + extractor_hash: ExtractorHash::from_hash(extractor_hash), + last_status_change: Utc::now().timestamp(), + parent_subscription_name: Some(sub.name.clone()), + priority: 0, + publish_date, + status: VideoStatus::Pick, + status_change: false, + thumbnail_url, + title: unwrap_option!(entry.title.clone()), + url, + }; + + println!("{}", video.to_color_display()); + add_video(app, video).await?; + } + + Ok(()) +} |