about summary refs log blame commit diff stats
path: root/src/update/mod.rs
blob: 9128bf76d6d3388afbedfde4cd71abaa627a40a1 (plain) (tree)













































































































































































































                                                                                              
// yt - A fully featured command line YouTube client
//
// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de>
// SPDX-License-Identifier: GPL-3.0-or-later
//
// This file is part of Yt.
//
// You should have received a copy of the License along with this program.
// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>.

use std::{collections::HashMap, process::Stdio, str::FromStr};

use anyhow::{Context, Ok, Result};
use chrono::{DateTime, Utc};
use log::{error, info, warn};
use tokio::{
    io::{AsyncBufReadExt, BufReader},
    process::Command,
};
use url::Url;
use yt_dlp::{unsmuggle_url, wrapper::info_json::InfoJson};

use crate::{
    app::App,
    storage::{
        subscriptions::{get_subscriptions, Subscription},
        video_database::{
            extractor_hash::ExtractorHash, getters::get_all_hashes, setters::add_video, Video,
            VideoStatus,
        },
    },
};

pub async fn update(
    app: &App,
    max_backlog: u32,
    subs_to_update: Vec<String>,
    _concurrent_processes: usize,
) -> Result<()> {
    let subscriptions = get_subscriptions(&app).await?;
    let mut back_subs: HashMap<Url, Subscription> = HashMap::new();

    let mut urls: Vec<String> = vec![];
    for (name, sub) in subscriptions.0 {
        if subs_to_update.contains(&name) || subs_to_update.is_empty() {
            urls.push(sub.url.to_string());
            back_subs.insert(sub.url.clone(), sub);
        } else {
            info!(
                "Not updating subscription '{}' as it was not specified",
                name
            );
        }
    }

    let mut child = Command::new("./python_update/raw_update.py")
        .arg(max_backlog.to_string())
        .args(&urls)
        .stdout(Stdio::piped())
        .stderr(Stdio::null())
        .stdin(Stdio::null())
        .spawn()
        .context("Failed to call python3 update_raw")?;

    let mut out = BufReader::new(
        child
            .stdout
            .take()
            .expect("Should be able to take child stdout"),
    )
    .lines();

    let hashes = get_all_hashes(app).await?;

    while let Some(line) = out.next_line().await? {
        // use tokio::{fs::File, io::AsyncWriteExt};
        // let mut output = File::create("output.json").await?;
        // output.write(line.as_bytes()).await?;
        // output.flush().await?;
        // output.sync_all().await?;
        // drop(output);

        let output_json: HashMap<Url, InfoJson> =
            serde_json::from_str(&line).expect("This should be valid json");

        for (url, value) in output_json {
            let sub = back_subs.get(&url).expect("This was stored before");
            process_subscription(app, sub, value, &hashes).await?
        }
    }

    let out = child.wait().await?;
    if out.success() {
        error!("A yt update-once invokation failed for all subscriptions.")
    }

    Ok(())
}

async fn process_subscription(
    app: &App,
    sub: &Subscription,
    entry: InfoJson,
    hashes: &Vec<blake3::Hash>,
) -> Result<()> {
    macro_rules! unwrap_option {
        ($option:expr) => {
            match $option {
                Some(x) => x,
                None => anyhow::bail!(concat!(
                    "Expected a value, but '",
                    stringify!($option),
                    "' is None!"
                )),
            }
        };
    }

    let publish_date = if let Some(date) = &entry.upload_date {
        let year: u32 = date
            .chars()
            .take(4)
            .collect::<String>()
            .parse()
            .expect("Should work.");
        let month: u32 = date
            .chars()
            .skip(4)
            .take(2)
            .collect::<String>()
            .parse()
            .expect("Should work");
        let day: u32 = date
            .chars()
            .skip(6)
            .take(2)
            .collect::<String>()
            .parse()
            .expect("Should work");

        let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z");
        Some(
            DateTime::<Utc>::from_str(&date_string)
                .expect("This should always work")
                .timestamp(),
        )
    } else {
        warn!(
            "The video '{}' lacks it's upload date!",
            unwrap_option!(&entry.title)
        );
        None
    };

    let thumbnail_url = match (&entry.thumbnails, &entry.thumbnail) {
        (None, None) => None,
        (None, Some(thumbnail)) => Some(thumbnail.to_owned()),

        // TODO: The algorithm is not exactly the best <2024-05-28>
        (Some(thumbnails), None) => Some(
            thumbnails
                .get(0)
                .expect("At least one should exist")
                .url
                .clone(),
        ),
        (Some(_), Some(thumnail)) => Some(thumnail.to_owned()),
    };

    let url = {
        let smug_url: url::Url = unwrap_option!(entry.webpage_url.clone());
        unsmuggle_url(smug_url)?
    };

    let extractor_hash = blake3::hash(url.as_str().as_bytes());

    if hashes.contains(&extractor_hash) {
        // We already stored the video information
        println!(
            "(Ignoring duplicated video from: '{}' -> '{}')",
            sub.name,
            unwrap_option!(entry.title)
        );
        return Ok(());
    } else {
        let video = Video {
            cache_path: None,
            description: entry.description.clone(),
            duration: entry.duration,
            extractor_hash: ExtractorHash::from_hash(extractor_hash),
            last_status_change: Utc::now().timestamp(),
            parent_subscription_name: Some(sub.name.clone()),
            priority: 0,
            publish_date,
            status: VideoStatus::Pick,
            status_change: false,
            thumbnail_url,
            title: unwrap_option!(entry.title.clone()),
            url,
        };

        println!("{}", video.to_color_display());
        add_video(app, video).await?;
    }

    Ok(())
}