diff options
author | Benedikt Peetz <benedikt.peetz@b-peetz.de> | 2025-02-16 09:57:23 +0100 |
---|---|---|
committer | Benedikt Peetz <benedikt.peetz@b-peetz.de> | 2025-02-16 09:57:23 +0100 |
commit | 6a137c6ca968654810ccfddd90908a227287387f (patch) | |
tree | d729b258ecc9fb8a58e108f5c534b3850d5e1011 | |
parent | feat(yt/update): Port the Python updater to rust (diff) | |
download | yt-6a137c6ca968654810ccfddd90908a227287387f.zip |
refactor(treewide): Remove all references of the now obsolete update_raw.py
-rw-r--r-- | flake.nix | 6 | ||||
-rw-r--r-- | package/blake3/add_cargo_lock.patch | 431 | ||||
-rw-r--r-- | package/blake3/add_cargo_lock.patch.license | 9 | ||||
-rw-r--r-- | package/blake3/blake3.nix | 38 | ||||
-rw-r--r-- | package/package.nix | 12 | ||||
-rwxr-xr-x | python_update/raw_update.py | 179 |
6 files changed, 5 insertions, 670 deletions
diff --git a/flake.nix b/flake.nix index abac232..c5a2fe6 100644 --- a/flake.nix +++ b/flake.nix @@ -32,7 +32,6 @@ python = pkgs.python3.withPackages (ps: [ ps.yt-dlp - blake3 ]); buildInputs = with pkgs; [ @@ -46,14 +45,13 @@ SDL2 ]; - yt = pkgs.callPackage ./package/package.nix {inherit blake3 tree-sitter-yts;}; - blake3 = pkgs.callPackage ./package/blake3/blake3.nix {}; + yt = pkgs.callPackage ./package/package.nix {inherit tree-sitter-yts;}; tree-sitter-yts = pkgs.callPackage ./tree-sitter-yts/package.nix {}; treefmtEval = import ./treefmt.nix {inherit treefmt-nix pkgs;}; in { packages = { - inherit yt blake3 tree-sitter-yts; + inherit yt tree-sitter-yts; default = self.packages.${system}.yt; }; diff --git a/package/blake3/add_cargo_lock.patch b/package/blake3/add_cargo_lock.patch deleted file mode 100644 index 19a5d1d..0000000 --- a/package/blake3/add_cargo_lock.patch +++ /dev/null @@ -1,431 +0,0 @@ -From 45fd97400c01f39f841f84d43f1d28f8102cd927 Mon Sep 17 00:00:00 2001 -From: Benedikt Peetz <benedikt.peetz@b-peetz.de> -Date: Thu, 22 Aug 2024 11:25:24 +0200 -Subject: [PATCH] build(cargo.lock): Add - ---- - Cargo.lock | 412 +++++++++++++++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 412 insertions(+) - create mode 100644 Cargo.lock - -diff --git a/Cargo.lock b/Cargo.lock -new file mode 100644 -index 0000000..98b4b7a ---- /dev/null -+++ b/Cargo.lock -@@ -0,0 +1,412 @@ -+# This file is automatically @generated by Cargo. -+# It is not intended for manual editing. -+version = 3 -+ -+[[package]] -+name = "arrayref" -+version = "0.3.8" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" -+ -+[[package]] -+name = "arrayvec" -+version = "0.7.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -+ -+[[package]] -+name = "autocfg" -+version = "1.3.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -+ -+[[package]] -+name = "bitflags" -+version = "2.6.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -+ -+[[package]] -+name = "blake3" -+version = "0.4.1" -+dependencies = [ -+ "blake3 1.5.4", -+ "hex", -+ "pyo3", -+ "rayon", -+] -+ -+[[package]] -+name = "blake3" -+version = "1.5.4" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" -+dependencies = [ -+ "arrayref", -+ "arrayvec", -+ "cc", -+ "cfg-if", -+ "constant_time_eq", -+ "memmap2", -+ "rayon-core", -+] -+ -+[[package]] -+name = "cc" -+version = "1.1.13" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" -+dependencies = [ -+ "shlex", -+] -+ -+[[package]] -+name = "cfg-if" -+version = "1.0.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -+ -+[[package]] -+name = "constant_time_eq" -+version = "0.3.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" -+ -+[[package]] -+name = "crossbeam-deque" -+version = "0.8.5" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -+dependencies = [ -+ "crossbeam-epoch", -+ "crossbeam-utils", -+] -+ -+[[package]] -+name = "crossbeam-epoch" -+version = "0.9.18" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -+dependencies = [ -+ "crossbeam-utils", -+] -+ -+[[package]] -+name = "crossbeam-utils" -+version = "0.8.20" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" -+ -+[[package]] -+name = "either" -+version = "1.13.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" -+ -+[[package]] -+name = "heck" -+version = "0.4.1" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -+ -+[[package]] -+name = "hex" -+version = "0.4.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -+ -+[[package]] -+name = "indoc" -+version = "2.0.5" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" -+ -+[[package]] -+name = "libc" -+version = "0.2.158" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" -+ -+[[package]] -+name = "lock_api" -+version = "0.4.12" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -+dependencies = [ -+ "autocfg", -+ "scopeguard", -+] -+ -+[[package]] -+name = "memmap2" -+version = "0.9.4" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" -+dependencies = [ -+ "libc", -+] -+ -+[[package]] -+name = "memoffset" -+version = "0.9.1" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -+dependencies = [ -+ "autocfg", -+] -+ -+[[package]] -+name = "once_cell" -+version = "1.19.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -+ -+[[package]] -+name = "parking_lot" -+version = "0.12.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -+dependencies = [ -+ "lock_api", -+ "parking_lot_core", -+] -+ -+[[package]] -+name = "parking_lot_core" -+version = "0.9.10" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -+dependencies = [ -+ "cfg-if", -+ "libc", -+ "redox_syscall", -+ "smallvec", -+ "windows-targets", -+] -+ -+[[package]] -+name = "portable-atomic" -+version = "1.7.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" -+ -+[[package]] -+name = "proc-macro2" -+version = "1.0.86" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" -+dependencies = [ -+ "unicode-ident", -+] -+ -+[[package]] -+name = "pyo3" -+version = "0.20.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" -+dependencies = [ -+ "cfg-if", -+ "indoc", -+ "libc", -+ "memoffset", -+ "parking_lot", -+ "portable-atomic", -+ "pyo3-build-config", -+ "pyo3-ffi", -+ "pyo3-macros", -+ "unindent", -+] -+ -+[[package]] -+name = "pyo3-build-config" -+version = "0.20.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" -+dependencies = [ -+ "once_cell", -+ "target-lexicon", -+] -+ -+[[package]] -+name = "pyo3-ffi" -+version = "0.20.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" -+dependencies = [ -+ "libc", -+ "pyo3-build-config", -+] -+ -+[[package]] -+name = "pyo3-macros" -+version = "0.20.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" -+dependencies = [ -+ "proc-macro2", -+ "pyo3-macros-backend", -+ "quote", -+ "syn", -+] -+ -+[[package]] -+name = "pyo3-macros-backend" -+version = "0.20.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" -+dependencies = [ -+ "heck", -+ "proc-macro2", -+ "pyo3-build-config", -+ "quote", -+ "syn", -+] -+ -+[[package]] -+name = "quote" -+version = "1.0.36" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -+dependencies = [ -+ "proc-macro2", -+] -+ -+[[package]] -+name = "rayon" -+version = "1.10.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -+dependencies = [ -+ "either", -+ "rayon-core", -+] -+ -+[[package]] -+name = "rayon-core" -+version = "1.12.1" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -+dependencies = [ -+ "crossbeam-deque", -+ "crossbeam-utils", -+] -+ -+[[package]] -+name = "redox_syscall" -+version = "0.5.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" -+dependencies = [ -+ "bitflags", -+] -+ -+[[package]] -+name = "scopeguard" -+version = "1.2.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -+ -+[[package]] -+name = "shlex" -+version = "1.3.0" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -+ -+[[package]] -+name = "smallvec" -+version = "1.13.2" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" -+ -+[[package]] -+name = "syn" -+version = "2.0.75" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" -+dependencies = [ -+ "proc-macro2", -+ "quote", -+ "unicode-ident", -+] -+ -+[[package]] -+name = "target-lexicon" -+version = "0.12.16" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" -+ -+[[package]] -+name = "unicode-ident" -+version = "1.0.12" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" -+ -+[[package]] -+name = "unindent" -+version = "0.2.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" -+ -+[[package]] -+name = "windows-targets" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -+dependencies = [ -+ "windows_aarch64_gnullvm", -+ "windows_aarch64_msvc", -+ "windows_i686_gnu", -+ "windows_i686_gnullvm", -+ "windows_i686_msvc", -+ "windows_x86_64_gnu", -+ "windows_x86_64_gnullvm", -+ "windows_x86_64_msvc", -+] -+ -+[[package]] -+name = "windows_aarch64_gnullvm" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -+ -+[[package]] -+name = "windows_aarch64_msvc" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -+ -+[[package]] -+name = "windows_i686_gnu" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -+ -+[[package]] -+name = "windows_i686_gnullvm" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -+ -+[[package]] -+name = "windows_i686_msvc" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -+ -+[[package]] -+name = "windows_x86_64_gnu" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -+ -+[[package]] -+name = "windows_x86_64_gnullvm" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -+ -+[[package]] -+name = "windows_x86_64_msvc" -+version = "0.52.6" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" --- -2.45.2 - diff --git a/package/blake3/add_cargo_lock.patch.license b/package/blake3/add_cargo_lock.patch.license deleted file mode 100644 index d4d410f..0000000 --- a/package/blake3/add_cargo_lock.patch.license +++ /dev/null @@ -1,9 +0,0 @@ -yt - A fully featured command line YouTube client - -Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -SPDX-License-Identifier: GPL-3.0-or-later - -This file is part of Yt. - -You should have received a copy of the License along with this program. -If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. diff --git a/package/blake3/blake3.nix b/package/blake3/blake3.nix deleted file mode 100644 index 3965112..0000000 --- a/package/blake3/blake3.nix +++ /dev/null @@ -1,38 +0,0 @@ -# yt - A fully featured command line YouTube client -# -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later -# -# This file is part of Yt. -# -# You should have received a copy of the License along with this program. -# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. -{ - python3Packages, - rustPlatform, - fetchFromGitHub, -}: -python3Packages.buildPythonPackage rec { - pname = "blake3"; - version = "0.4.1"; - - src = fetchFromGitHub { - owner = "oconnor663"; - repo = "blake3-py"; - rev = version; - hash = "sha256-Ju40ea8IQMOPg9BiN47BMmr/WU8HptbqqzVI+jNGpA8="; - }; - - patches = [ - ./add_cargo_lock.patch - ]; - - cargoDeps = rustPlatform.fetchCargoVendor { - inherit src patches; - hash = "sha256-Woz0f869td1SyRaVw6qVpBW4aJC/5M/tkmxgLR6HDN4="; - }; - - format = "pyproject"; - - nativeBuildInputs = with rustPlatform; [cargoSetupHook maturinBuildHook]; -} diff --git a/package/package.nix b/package/package.nix index a5c33a7..11e63b2 100644 --- a/package/package.nix +++ b/package/package.nix @@ -17,7 +17,6 @@ python3, rustPlatform, sqlite, - blake3, tree-sitter-yts, }: let version = "0.1.0"; @@ -25,7 +24,7 @@ src = ./..; buildInputs = [ - (python3.withPackages (ps: [ps.yt-dlp blake3])) + (python3.withPackages (ps: [ps.yt-dlp])) mpv-unwrapped.dev ffmpeg ]; @@ -60,20 +59,15 @@ in ''; passthru = { - inherit blake3 tree-sitter-yts; + inherit tree-sitter-yts; }; cargoLock = { lockFile = ../Cargo.lock; }; - postBuild = '' - install -m755 ./python_update/raw_update.py -D "$out/bin/raw_update.py" - patchShebangs "$out/bin/raw_update.py" - ''; - postInstall = '' wrapProgram $out/bin/yt \ - --prefix PATH : ${lib.makeBinPath buildInputs}:$out/bin + --prefix PATH : ${lib.makeBinPath buildInputs} ''; } diff --git a/python_update/raw_update.py b/python_update/raw_update.py deleted file mode 100755 index 28a2bac..0000000 --- a/python_update/raw_update.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python - -# yt - A fully featured command line YouTube client -# -# Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de> -# SPDX-License-Identifier: GPL-3.0-or-later -# -# This file is part of Yt. -# -# You should have received a copy of the License along with this program. -# If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>. - -# This has been take from the `ytcc` updater code (at `8893bc98428cb78d458a9cf3ded03f519d86a46b`). -# Source URL: https://github.com/woefe/ytcc/commit/8893bc98428cb78d458a9cf3ded03f519d86a46b - -from blake3 import blake3 -from dataclasses import dataclass -from functools import partial -from typing import Any, Iterable, Optional, Tuple, TypeVar -import asyncio -import itertools -import json -import logging -import sys - -import yt_dlp - - -@dataclass(frozen=True) -class Playlist: - name: str - url: str - reverse: bool - - -@dataclass(frozen=True) -class Video: - url: str - title: str - description: str - publish_date: float - watch_date: Optional[float] - duration: float - thumbnail_url: Optional[str] - extractor_hash: str - - @property - def watched(self) -> bool: - return self.watch_date is not None - - -logger = logging.getLogger("yt") -logging.basicConfig(encoding="utf-8", level=int(sys.argv[3])) - -_ytdl_logger = logging.getLogger("yt_dlp") -_ytdl_logger.propagate = False -_ytdl_logger.addHandler(logging.NullHandler()) -YTDL_COMMON_OPTS = {"logger": _ytdl_logger} - -T = TypeVar("T") - - -def take(amount: int, iterable: Iterable[T]) -> Iterable[T]: - """Take the first elements of an iterable. - - If the given iterable has less elements than the given amount, the returned iterable has the - same amount of elements as the given iterable. Otherwise the returned iterable has `amount` - elements. - - :param amount: The number of elements to take - :param iterable: The iterable to take elements from - :return: The first elements of the given iterable - """ - for _, elem in zip(range(amount), iterable): - yield elem - - -class Fetcher: - def __init__(self, max_backlog): - self.max_items = max_backlog - self.ydl_opts = { - **YTDL_COMMON_OPTS, - "playliststart": 1, - "playlistend": max_backlog, - "noplaylist": False, - "extractor_args": {"youtubetab": {"approximate_date": [""]}}, - } - - async def get_unprocessed_entries( - self, url: str, hashes: Iterable[str] - ) -> Iterable[Tuple[str, str, Any]]: - result = [] - with yt_dlp.YoutubeDL(self.ydl_opts) as ydl: - logger.info("Checking playlist '%s'...", url) - try: - loop = asyncio.get_event_loop() - info = await loop.run_in_executor( - None, - partial(ydl.extract_info, url, download=False, process=False), - ) - except yt_dlp.DownloadError as download_error: - logger.error( - "Failed to get playlist '%s'. Error was: '%s'", - url, - download_error, - ) - else: - entries = info.get("entries", []) - for entry in take(self.max_items, entries): - logger.debug(json.dumps(entry)) - id = str.encode(yt_dlp.utils.unsmuggle_url(entry["id"])[0]) - ehash = blake3(id).hexdigest() - if ehash not in hashes: - result.append((url, entry)) - return result - - def _process_ie(self, entry): - with yt_dlp.YoutubeDL(self.ydl_opts) as ydl: - processed = ydl.process_ie_result(entry, False) - - return { - "description": processed.get("description"), - "duration": processed.get("duration"), - "upload_date": processed.get("upload_date"), - "thumbnails": processed.get("thumbnails"), - "thumbnail": processed.get("thumbnail"), - "title": processed.get("title"), - "webpage_url": processed.get("webpage_url"), - "id": processed.get("id"), - } - - async def process_entry(self, url: str, entry: Any) -> Optional[Any]: - try: - loop = asyncio.get_event_loop() - processed = await loop.run_in_executor(None, self._process_ie, entry) - except yt_dlp.DownloadError as download_error: - logger.error( - "Failed to get a video of playlist '%s'. Error was: '%s'", - url, - download_error, - ) - return None - else: - print(json.dumps({url: processed})) - - -class Updater: - def __init__(self, max_backlog=20): - self.max_items = max_backlog - self.fetcher = Fetcher(max_backlog) - self.hashes = None - - async def update_url(self, url: str): - logger.info(f"Updating {url}...") - new_entries = await self.fetcher.get_unprocessed_entries(url, self.hashes) - - await asyncio.gather( - *itertools.starmap(self.fetcher.process_entry, new_entries) - ) - - async def do_update(self, urls: Iterable[str]): - await asyncio.gather(*map(self.update_url, urls)) - - def update(self, urls: Iterable[str], hashes: Iterable[str]): - self.hashes = hashes - asyncio.run(self.do_update(urls)) - - -def update(): - max_backlog = int(sys.argv[1]) - subscriptions_number = int(sys.argv[2]) - u = Updater(max_backlog=max_backlog) - u.update( - sys.argv[4 : (4 + subscriptions_number)], sys.argv[(4 + subscriptions_number) :] - ) - - -logger.debug(sys.argv) -update() |