// yt - A fully featured command line YouTube client
//
// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de>
// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of Yt.
//
// You should have received a copy of the License along with this program.
// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>.
use std::{collections::HashMap, process::Stdio, str::FromStr};
use anyhow::{Context, Ok, Result};
use chrono::{DateTime, Utc};
use log::{error, info, warn};
use tokio::{
io::{AsyncBufReadExt, BufReader},
process::Command,
};
use url::Url;
use yt_dlp::{unsmuggle_url, wrapper::info_json::InfoJson};
use crate::{
app::App,
storage::{
subscriptions::{get_subscriptions, Subscription},
video_database::{
extractor_hash::ExtractorHash, getters::get_all_hashes, setters::add_video, Video,
VideoStatus,
},
},
};
pub async fn update(
app: &App,
max_backlog: u32,
subs_to_update: Vec<String>,
_concurrent_processes: usize,
) -> Result<()> {
let subscriptions = get_subscriptions(&app).await?;
let mut back_subs: HashMap<Url, Subscription> = HashMap::new();
let mut urls: Vec<String> = vec![];
for (name, sub) in subscriptions.0 {
if subs_to_update.contains(&name) || subs_to_update.is_empty() {
urls.push(sub.url.to_string());
back_subs.insert(sub.url.clone(), sub);
} else {
info!(
"Not updating subscription '{}' as it was not specified",
name
);
}
}
let mut child = Command::new("./python_update/raw_update.py")
.arg(max_backlog.to_string())
.args(&urls)
.stdout(Stdio::piped())
.stderr(Stdio::null())
.stdin(Stdio::null())
.spawn()
.context("Failed to call python3 update_raw")?;
let mut out = BufReader::new(
child
.stdout
.take()
.expect("Should be able to take child stdout"),
)
.lines();
let hashes = get_all_hashes(app).await?;
while let Some(line) = out.next_line().await? {
// use tokio::{fs::File, io::AsyncWriteExt};
// let mut output = File::create("output.json").await?;
// output.write(line.as_bytes()).await?;
// output.flush().await?;
// output.sync_all().await?;
// drop(output);
let output_json: HashMap<Url, InfoJson> =
serde_json::from_str(&line).expect("This should be valid json");
for (url, value) in output_json {
let sub = back_subs.get(&url).expect("This was stored before");
process_subscription(app, sub, value, &hashes).await?
}
}
let out = child.wait().await?;
if out.success() {
error!("A yt update-once invokation failed for all subscriptions.")
}
Ok(())
}
async fn process_subscription(
app: &App,
sub: &Subscription,
entry: InfoJson,
hashes: &Vec<blake3::Hash>,
) -> Result<()> {
macro_rules! unwrap_option {
($option:expr) => {
match $option {
Some(x) => x,
None => anyhow::bail!(concat!(
"Expected a value, but '",
stringify!($option),
"' is None!"
)),
}
};
}
let publish_date = if let Some(date) = &entry.upload_date {
let year: u32 = date
.chars()
.take(4)
.collect::<String>()
.parse()
.expect("Should work.");
let month: u32 = date
.chars()
.skip(4)
.take(2)
.collect::<String>()
.parse()
.expect("Should work");
let day: u32 = date
.chars()
.skip(6)
.take(2)
.collect::<String>()
.parse()
.expect("Should work");
let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z");
Some(
DateTime::<Utc>::from_str(&date_string)
.expect("This should always work")
.timestamp(),
)
} else {
warn!(
"The video '{}' lacks it's upload date!",
unwrap_option!(&entry.title)
);
None
};
let thumbnail_url = match (&entry.thumbnails, &entry.thumbnail) {
(None, None) => None,
(None, Some(thumbnail)) => Some(thumbnail.to_owned()),
// TODO: The algorithm is not exactly the best <2024-05-28>
(Some(thumbnails), None) => Some(
thumbnails
.get(0)
.expect("At least one should exist")
.url
.clone(),
),
(Some(_), Some(thumnail)) => Some(thumnail.to_owned()),
};
let url = {
let smug_url: url::Url = unwrap_option!(entry.webpage_url.clone());
unsmuggle_url(smug_url)?
};
let extractor_hash = blake3::hash(url.as_str().as_bytes());
if hashes.contains(&extractor_hash) {
// We already stored the video information
println!(
"(Ignoring duplicated video from: '{}' -> '{}')",
sub.name,
unwrap_option!(entry.title)
);
return Ok(());
} else {
let video = Video {
cache_path: None,
description: entry.description.clone(),
duration: entry.duration,
extractor_hash: ExtractorHash::from_hash(extractor_hash),
last_status_change: Utc::now().timestamp(),
parent_subscription_name: Some(sub.name.clone()),
priority: 0,
publish_date,
status: VideoStatus::Pick,
status_change: false,
thumbnail_url,
title: unwrap_option!(entry.title.clone()),
url,
};
println!("{}", video.to_color_display());
add_video(app, video).await?;
}
Ok(())
}