diff options
Diffstat (limited to 'pkgs/by-name/up/update-vim-plugins/update_vim_plugins')
13 files changed, 1190 insertions, 0 deletions
diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/__init__.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/__init__.py diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/__main__.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/__main__.py new file mode 100644 index 00000000..a8d9e06f --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/__main__.py @@ -0,0 +1,15 @@ +from cleo.application import Application + +from .update import UpdateCommand +from .cleanup import CleanUpCommand + + +def main(): + application = Application() + application.add(UpdateCommand()) + application.add(CleanUpCommand()) + application.run() + + +if __name__ == "__main__": + main() diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/cleanup.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/cleanup.py new file mode 100644 index 00000000..fd313ed0 --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/cleanup.py @@ -0,0 +1,100 @@ +from cleo.commands.command import Command +from cleo.helpers import argument + +from .helpers import read_manifest_to_spec, read_blacklist_to_spec, write_manifest_from_spec + + +class CleanUpCommand(Command): + name = "cleanup" + description = "Clean up manifest" + arguments = [argument("plug_dir", description="Path to the plugin directory", optional=False)] + + def handle(self): + """Main command function""" + + plug_dir = self.argument("plug_dir") + self.line("<comment>Checking manifest file</comment>") + # all cleaning up will be done during reading and writing automatically + manifest = read_manifest_to_spec(plug_dir) + blacklist = read_blacklist_to_spec(plug_dir) + + new_manifest = [spec for spec in manifest if spec not in blacklist] + + new_manifest_filterd = self.filter_renamed(new_manifest) + + write_manifest_from_spec(new_manifest_filterd, plug_dir) + + self.line("<comment>Done</comment>") + + def filter_renamed(self, specs): + """Filter specs that define the same plugin (same owner and same repo) but with different properties. + This could be a different name, source, or branch + """ + + error = False + for i, p in enumerate(specs): + for p2 in specs: + same_owner = p.owner.lower() == p2.owner.lower() + same_repo = p.repo.lower() == p2.repo.lower() + different_specs = p != p2 + marked_duplicate = p.marked_duplicate or p2.marked_duplicate + + if same_owner and same_repo and different_specs and not marked_duplicate: + self.line("<info>The following lines appear to define the same plugin</info>") + + p_props_defined = p.branch is not None or p.custom_name is not None + p2_props_defined = p2.branch is not None or p2.custom_name is not None + p_is_lower_case = p.owner == p.owner.lower() and p.name == p.name.lower() + p2_is_lower_case = p2.owner == p2.owner.lower() and p2.name == p2.name.lower() + + # list of conditions for selecting p + select_p = p_props_defined and not p2_props_defined or p2_is_lower_case and not p_is_lower_case + # list of conditions for selecting p2 + select_p2 = p2_props_defined and not p_props_defined or p_is_lower_case and not p2_is_lower_case + + # one is more defined and is all lower, but the other is not all lower + # (we assume the not all lower case is the correct naming) + error_props_lower = ( + p_props_defined and p_is_lower_case and not p2_props_defined and not p2_is_lower_case + ) + error_props_lower2 = ( + p2_props_defined and p2_is_lower_case and not p_props_defined and not p_is_lower_case + ) + + # both props are defined + error_props = p_props_defined and p2_props_defined + + # the sources are different + error_source = p.repository_host != p2.repository_host + + if error_props_lower or error_props_lower2 or error_props or error_source: + self.line(" • <error>Cannot determine which is the correct plugin</error>") + self.line(f" - {p.line}") + self.line(f" - {p2.line}") + error = True + # remove second spec to not encounter the error twice + # this will not be written to the manifest.txt because we set + # the error flag and will exit after the loop + specs.remove(p2) + elif select_p: + self.line(f" - <comment>{p.line}</comment>") + self.line(f" - {p2.line}") + specs.remove(p2) + elif select_p2: + self.line(f" - {p.line}") + self.line(f" - <comment>{p2.line}</comment>") + specs.remove(p) + else: + self.line(" • <error>Logic error in correct spec determination</error>") + self.line(f" - {p.line}") + self.line(f" - {p2.line}") + error = True + # remove second spec to not encounter the error twice + # this will not be written to the manifest.txt because we set + # the error flag and will exit after the loop + specs.remove(p) + if error: + # exit after all errors have been found + exit(1) + + return specs diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/helpers.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/helpers.py new file mode 100644 index 00000000..8a28b0e8 --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/helpers.py @@ -0,0 +1,61 @@ +from .spec import PluginSpec + +MANIFEST_FILE = "manifest.txt" +BLACKLIST_FILE = "blacklist.txt" +PKGS_FILE = "default.nix" +JSON_FILE = ".plugins.json" +PLUGINS_LIST_FILE = "plugins.md" + + +def get_const(const: str, plug_dir: str) -> str: + out = plug_dir + "/" + const + return out + + +def read_manifest(plug_dir: str) -> list[str]: + with open(get_const(MANIFEST_FILE, plug_dir), "r") as file: + specs = set([spec.strip() for spec in file.readlines()]) + + return sorted(specs) + + +def read_manifest_to_spec(plug_dir: str) -> list[PluginSpec]: + manifest = read_manifest(plug_dir) + specs = [PluginSpec.from_spec(spec.strip()) for spec in manifest] + + return sorted(specs) + + +def read_blacklist(plug_dir: str) -> list[str]: + with open(get_const(BLACKLIST_FILE, plug_dir), "r") as file: + if len(file.readlines()) == 0: + return [""] + else: + blacklisted_specs = set([spec.strip() for spec in file.readlines()]) + + return sorted(blacklisted_specs) + + +def read_blacklist_to_spec(plug_dir: str) -> list[PluginSpec]: + blacklist = read_blacklist(plug_dir) + specs = [PluginSpec.from_spec(spec.strip()) for spec in blacklist] + + return sorted(specs) + + +def write_manifest(specs: list[str] | set[str], plug_dir: str): + """write specs to manifest file. Does some cleaning up""" + + with open(get_const(MANIFEST_FILE, plug_dir), "w") as file: + specs = sorted(set(specs), key=lambda x: x.lower()) + specs = [p for p in specs] + + for s in specs: + file.write(f"{s}\n") + + +def write_manifest_from_spec(specs: list[PluginSpec], plug_dir: str): + """write specs to manifest file. Does some cleaning up""" + + strings = [f"{spec}" for spec in specs] + write_manifest(strings, plug_dir) diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/nix.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/nix.py new file mode 100644 index 00000000..66a8df4c --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/nix.py @@ -0,0 +1,121 @@ +import abc +import enum +import json +import subprocess + + +def nix_prefetch_url(url): + """Return the sha256 hash of the given url.""" + subprocess_output = subprocess.check_output( + ["nix-prefetch-url", "--type", "sha256", url], + stderr=subprocess.DEVNULL, + ) + sha256 = subprocess_output.decode("utf-8").strip() + return sha256 + + +def nix_prefetch_git(url): + """Return the sha256 hash of the given git url.""" + subprocess_output = subprocess.check_output(["nix-prefetch-git", url], stderr=subprocess.DEVNULL) + sha256 = json.loads(subprocess_output)["sha256"] + return sha256 + + +class Source(abc.ABC): + """Abstract base class for sources.""" + + url: str + sha256: str + + @abc.abstractmethod + def __init__(self, url: str) -> None: + """Initialize a Source.""" + + @abc.abstractmethod + def get_nix_expression(self): + """Return the nix expression for this source.""" + + def __repr__(self): + """Return the representation of this source.""" + return self.get_nix_expression() + + +class UrlSource(Source): + """A source that is a url.""" + + def __init__(self, url: str) -> None: + """Initialize a UrlSource.""" + self.url = url + self.sha256 = nix_prefetch_url(url) + + def get_nix_expression(self): + """Return the nix expression for this source.""" + return f'fetchurl {{ url = "{self.url}"; sha256 = "{self.sha256}"; }}' + + +class GitSource(Source): + """A source that is a git repository.""" + + def __init__(self, url: str, rev: str) -> None: + """Initialize a GitSource.""" + self.url = url + self.rev = rev + self.sha256 = nix_prefetch_git(url) + + def get_nix_expression(self): + """Return the nix expression for this source.""" + return f'fetchgit {{ url = "{self.url}"; rev = "{self.rev}"; sha256 = "{self.sha256}"; }}' + + +class License(enum.Enum): + """An enumeration of licenses.""" + + AGPL_3_0 = "agpl3Only" + APACHE_2_0 = "asf20" + BSD_2_CLAUSE = "bsd2" + BSD_3_CLAUSE = "bsd3" + BSL_1_0 = "bsl1_0" + CC0_1_0 = "cc0" + EPL_2_0 = "epl20" + GPL_2_0 = "gpl2Only" + GPL_3_0 = "gpl3Only" + ISCLGPL_2_1 = "lgpl21Only" + MIT = "mit" + MPL_2_0 = "mpl20" + UNLUNLICENSE = "unlicense" + WTFPL = "wtfpl" + UNFREE = "unfree" + UNKNOWN = "" + + @classmethod + def from_spdx_id(cls, spdx_id: str | None) -> "License": + """Return the License from the given spdx_id.""" + mapping = { + "AGPL-3.0": cls.AGPL_3_0, + "AGPL-3.0-only": cls.AGPL_3_0, + "Apache-2.0": cls.APACHE_2_0, + "BSD-2-Clause": cls.BSD_2_CLAUSE, + "BSD-3-Clause": cls.BSD_3_CLAUSE, + "BSL-1.0": cls.BSL_1_0, + "CC0-1.0": cls.CC0_1_0, + "EPL-2.0": cls.EPL_2_0, + "GPL-2.0": cls.GPL_2_0, + "GPL-2.0-only": cls.GPL_2_0, + "GPL-3.0": cls.GPL_3_0, + "GPL-3.0-only": cls.GPL_3_0, + "LGPL-2.1-only": cls.ISCLGPL_2_1, + "MIT": cls.MIT, + "MPL-2.0": cls.MPL_2_0, + "Unlicense": cls.UNLUNLICENSE, + "WTFPL": cls.WTFPL, + } + + if spdx_id is None: + return cls.UNKNOWN + + spdx_id = spdx_id.upper() + return mapping.get(spdx_id, cls.UNKNOWN) + + def __str__(self): + """Return the string representation of this license.""" + return self.value diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/plugin.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/plugin.py new file mode 100644 index 00000000..8334ad53 --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/plugin.py @@ -0,0 +1,182 @@ +import logging +import os +import urllib + +import requests +import jsonpickle +from datetime import datetime, date +from dateparser import parse + +from .nix import GitSource, License, Source, UrlSource +from .spec import PluginSpec, RepositoryHost + + +logger = logging.getLogger(__name__) + + +class VimPlugin: + """Abstract base class for vim plugins.""" + + name: str + owner: str + repo: str + version: date + source: Source + description: str = "No description" + homepage: str + license: License + source_line: str + checked: date = datetime.now().date() + + def to_nix(self): + """Return the nix expression for this plugin.""" + meta = f'with lib; {{ description = "{self.description}"; homepage = "{self.homepage}"; license = with licenses; [ {self.license.value} ]; }}' + return f'/* Generated from: {self.source_line} */ {self.name} = buildVimPlugin {{ pname = "{self.name}"; version = "{self.version}"; src = {self.source.get_nix_expression()}; meta = {meta}; }};' + + def to_json(self): + """Serizalize the plugin to json""" + return jsonpickle.encode(self) + + def to_markdown(self): + link = f"[{self.source_line}]({self.homepage})" + version = f"{self.version}" + package_name = f"{self.name}" + checked = f"{self.checked}" + + return f"| {link} | {version} | `{package_name}` | {checked} |" + + def __lt__(self, o: object) -> bool: + if not isinstance(o, VimPlugin): + return False + + return self.name.lower() < o.name.lower() + + def __repr__(self): + """Return the representation of this plugin.""" + return f"VimPlugin({self.name!r}, {self.version.strftime('%Y-%m-%d')})" + + +def _get_github_token(): + token = os.environ.get("GITHUB_TOKEN") + if token is None: + # NOTE: This should never use more than the free api requests <2023-12-09> + pass + # logger.warning("GITHUB_TOKEN environment variable not set") + return token + + +class GitHubPlugin(VimPlugin): + def __init__(self, plugin_spec: PluginSpec) -> None: + """Initialize a GitHubPlugin.""" + + full_name = f"{plugin_spec.owner}/{plugin_spec.repo}" + repo_info = self._api_call(f"repos/{full_name}") + default_branch = plugin_spec.branch or repo_info["default_branch"] + api_callback = self._api_call(f"repos/{full_name}/commits/{default_branch}") + latest_commit = api_callback["commit"] + sha = api_callback["sha"] + + self.name = plugin_spec.name + self.owner = plugin_spec.owner + self.version = parse(latest_commit["committer"]["date"]).date() + self.source = UrlSource(f"https://github.com/{full_name}/archive/{sha}.tar.gz") + self.description = (repo_info.get("description") or "").replace('"', '\\"') + self.homepage = repo_info["html_url"] + self.license = plugin_spec.license or License.from_spdx_id((repo_info.get("license") or {}).get("spdx_id")) + self.source_line = plugin_spec.line + + def _api_call(self, path: str, token: str | None = _get_github_token()): + """Call the GitHub API.""" + url = f"https://api.github.com/{path}" + headers = {"Content-Type": "application/json"} + if token is not None: + headers["Authorization"] = f"token {token}" + response = requests.get(url, headers=headers) + if response.status_code != 200: + raise RuntimeError(f"GitHub API call failed: {response.text}") + return response.json() + + +class GitlabPlugin(VimPlugin): + def __init__(self, plugin_spec: PluginSpec) -> None: + """Initialize a GitlabPlugin.""" + + full_name = urllib.parse.quote(f"{plugin_spec.owner}/{plugin_spec.repo}", safe="") + repo_info = self._api_call(f"projects/{full_name}") + default_branch = plugin_spec.branch or repo_info["default_branch"] + api_callback = self._api_call(f"projects/{full_name}/repository/branches/{default_branch}") + latest_commit = api_callback["commit"] + sha = latest_commit["id"] + + self.name = plugin_spec.name + self.owner = plugin_spec.owner + self.version = parse(latest_commit["created_at"]).date() + self.source = UrlSource(f"https://gitlab.com/api/v4/projects/{full_name}/repository/archive.tar.gz?sha={sha}") + self.description = (repo_info.get("description") or "").replace('"', '\\"') + self.homepage = repo_info["web_url"] + self.license = plugin_spec.license or License.from_spdx_id(repo_info.get("license", {}).get("key")) + self.source_line = plugin_spec.line + + def _api_call(self, path: str) -> dict: + """Call the Gitlab API.""" + url = f"https://gitlab.com/api/v4/{path}" + response = requests.get(url) + if response.status_code != 200: + raise RuntimeError(f"Gitlab API call failed: {response.text}") + return response.json() + + +def _get_sourcehut_token(): + token = os.environ.get("SOURCEHUT_TOKEN") + if token is None: + # NOTE: This should never use more than the free requests <2023-12-09> + pass + # logger.warning("SOURCEHUT_TOKEN environment variable not set") + return token + + +class SourceHutPlugin(VimPlugin): + def __init__(self, plugin_spec: PluginSpec) -> None: + """Initialize a SourceHutPlugin.""" + + repo_info = self._api_call(f"~{plugin_spec.owner}/repos/{plugin_spec.repo}") + if plugin_spec.branch is None: + commits = self._api_call(f"~{plugin_spec.owner}/repos/{plugin_spec.repo}/log") + else: + commits = self._api_call(f"~{plugin_spec.owner}/repos/{plugin_spec.repo}/log/{plugin_spec.branch}") + latest_commit = commits["results"][0] + sha = latest_commit["id"] + + self.name = plugin_spec.name + self.owner = plugin_spec.owner + self.version = parse(latest_commit["timestamp"]).date() + self.description = (repo_info.get("description") or "").replace('"', '\\"') + self.homepage = f"https://git.sr.ht/~{plugin_spec.owner}/{plugin_spec.repo}" + self.source = GitSource(self.homepage, sha) + self.license = plugin_spec.license or License.UNKNOWN # cannot be determined via API + self.source_line = plugin_spec.line + + def _api_call(self, path: str, token: str | None = _get_sourcehut_token()): + """Call the SourceHut API.""" + + url = f"https://git.sr.ht/api/{path}" + headers = {"Content-Type": "application/json"} + if token is not None: + headers["Authorization"] = f"token {token}" + response = requests.get(url, headers=headers) + if response.status_code != 200: + raise RuntimeError(f"SourceHut API call failed: {response.json()}") + return response.json() + + +def plugin_from_spec(plugin_spec: PluginSpec) -> VimPlugin: + """Initialize a VimPlugin.""" + + if plugin_spec.repository_host == RepositoryHost.GITHUB: + return GitHubPlugin(plugin_spec) + elif plugin_spec.repository_host == RepositoryHost.GITLAB: + return GitlabPlugin(plugin_spec) + elif plugin_spec.repository_host == RepositoryHost.SOURCEHUT: + return SourceHutPlugin(plugin_spec) + else: + raise NotImplementedError(f"Unsupported source: {plugin_spec.repository_host}") diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/spec.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/spec.py new file mode 100644 index 00000000..0f2fb29c --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/spec.py @@ -0,0 +1,143 @@ +import enum +import re + +from .nix import License + + +class RepositoryHost(enum.Enum): + """A repository host.""" + + GITHUB = "github" + GITLAB = "gitlab" + SOURCEHUT = "sourcehut" + + +class PluginSpec: + """A Vim plugin Spec.""" + + @classmethod + def from_spec(cls, spec): + """The spec line must be in the format: + [<repository_host>:]<owner>/<repo>[:<branch>][:name]. + + repository_host is one of github (default), gitlab, or sourcehut. + owner is the repository owner. + repo is the repository name. + branch is the git branch. + name is the name to use for the plugin (default is value of repo). + """ + repository_host = RepositoryHost.GITHUB + # gitref = "master" + + repository_host_regex = r"((?P<repository_host>[^:]+):)" + owner_regex = r"(?P<owner>[^/:]+)" + repo_regex = r"(?P<repo>[^:]+)" + branch_regex = r"(:(?P<branch>[^:]+)?)" + name_regex = r"(:(?P<name>[^:]+)?)" + license_regex = r"(:(?P<license>[^:]+)?)" + marked_duplicate_regex = r"(:(?P<duplicate>duplicate))" + + spec_regex = re.compile( + f"^{repository_host_regex}?{owner_regex}/{repo_regex}{branch_regex}?{name_regex}?{license_regex}?{marked_duplicate_regex}?$", + ) + + match = spec_regex.match(spec) + if match is None: + raise ValueError(f"Invalid spec: {spec}") + + group_dict = match.groupdict() + + repository_host = RepositoryHost(group_dict.get("repository_host") or "github") + + owner = group_dict.get("owner") + if owner is None: + raise RuntimeError("Could not get owner") + + repo = group_dict.get("repo") + if repo is None: + raise RuntimeError("Could not get repo") + + branch = group_dict.get("branch") + name = group_dict.get("name") + license = group_dict.get("license") + marked_duplicate = bool(group_dict.get("duplicate")) # True if 'duplicate', False if None + + line = spec + + return cls(repository_host, owner, repo, line, branch, name, license, marked_duplicate) + + def __init__( + self, + repository_host: RepositoryHost, + owner: str, + repo: str, + line: str, + branch: str | None = None, + name: str | None = None, + license: str | None = None, + marked_duplicate: bool = False, + ) -> None: + """Initialize a VimPluginSpec.""" + self.repository_host = repository_host + self.owner = owner + self.repo = repo + self.branch = branch + self.custom_name = name + self.name = name or repo.replace(".", "-").replace("_", "-") + self.license = License(license) if license else None + self.line = line + self.marked_duplicate = marked_duplicate + + def __str__(self) -> str: + """Return a string representation of a VimPluginSpec.""" + spec = "" + + if self.repository_host != RepositoryHost.GITHUB: + spec += f"{self.repository_host.value}:" + + spec += f"{self.owner}/{self.repo}" + + spec += ":" + if self.branch is not None: + spec += self.branch + + spec += ":" + if self.custom_name is not None: + spec += self.custom_name + + spec += ":" + if self.license is not None: + spec += str(self.license) + + spec += ":" + if self.marked_duplicate: + spec += "duplicate" + + return spec.rstrip(":") + + def __repr__(self): + """Return the representation of the specs""" + return f"PluginSpec({self.owner}/{self.repo}, {self.name})" + + def to_spec(self): + """Return a spec line for a VimPluginSpec.""" + return str(self) + + def __lt__(self, o: object) -> bool: + if not isinstance(o, PluginSpec): + return False + + return self.name.lower() < o.name.lower() + + def __eq__(self, o: object) -> bool: + """Return True if the two specs are equal.""" + if not isinstance(o, PluginSpec): + return False + + return ( + self.repository_host == o.repository_host + and self.owner == o.owner + and self.repo == o.repo + and self.branch == o.branch + and self.name == o.name + ) diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/__init__.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/__init__.py diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/fixtures.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/fixtures.py new file mode 100644 index 00000000..75dd251a --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/fixtures.py @@ -0,0 +1,44 @@ +import json + +import pytest +from pytest_mock import MockerFixture + +from update_vim_plugins.nix import GitSource, UrlSource + + +@pytest.fixture() +def url(): + return "https://example.com" + + +@pytest.fixture() +def rev(): + return "1234567890abcdef" + + +@pytest.fixture() +def sha256(): + return "sha256-1234567890abcdef" + + +@pytest.fixture() +def url_source(mocker: MockerFixture, url: str, sha256: str): + mocker.patch("subprocess.check_output", return_value=bytes(sha256, "utf-8")) + return UrlSource(url) + + +@pytest.fixture() +def git_source(mocker: MockerFixture, url: str, rev: str, sha256: str): + return_value = { + "url": url, + "rev": rev, + "date": "1970-01-01T00:00:00+00:00", + "path": "", + "sha256": sha256, + "fetchLFS": False, + "fetchSubmodules": False, + "deepClone": False, + "leaveDotGit": False, + } + mocker.patch("subprocess.check_output", return_value=json.dumps(return_value).encode("utf-8")) + return GitSource(url, rev) diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_nix.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_nix.py new file mode 100644 index 00000000..46e59f76 --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_nix.py @@ -0,0 +1,32 @@ +from update_vim_plugins.nix import GitSource, License, UrlSource + + +def test_url_source(url_source: UrlSource, url: str, sha256: str): + assert url_source.url == url + assert url_source.sha256 == sha256 + + +def test_url_source_nix_expression(url_source: UrlSource, url: str, sha256: str): + assert url_source.get_nix_expression() == f'fetchurl {{ url = "{url}"; sha256 = "{sha256}"; }}' + + +def test_git_source(git_source: GitSource, url: str, rev: str, sha256: str): + assert git_source.url == url + assert git_source.sha256 == sha256 + assert git_source.rev == rev + + +def test_git_source_nix_expression(git_source: GitSource, url: str, rev: str, sha256: str): + assert git_source.get_nix_expression() == f'fetchgit {{ url = "{url}"; rev = "{rev}"; sha256 = "{sha256}"; }}' + + +def test_license_github(): + github_license = "MIT" + license = License.from_spdx_id(github_license) + assert license == License.MIT + + +def test_license_gitlab(): + gitlab_license = "mit" + license = License.from_spdx_id(gitlab_license) + assert license == License.MIT diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_plugin.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_plugin.py new file mode 100644 index 00000000..32377e24 --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_plugin.py @@ -0,0 +1,144 @@ +import json +from typing import Callable + +import pytest +from pytest_mock import MockFixture + +from update_vim_plugins.nix import License, UrlSource +from update_vim_plugins.plugin import GitHubPlugin, VimPlugin +from update_vim_plugins.spec import PluginSpec + + +@pytest.fixture() +def mock_source(sha256: str): + class MockSource: + def __init__(self, *args, **kwargs): + pass + + def get_nix_expression(self): + return "src" + + return MockSource() + + +@pytest.fixture() +def mock_plugin(mock_source): + class MockVimPlugin(VimPlugin): + def __init__(self): + self.name = "test" + self.version = "1.0.0" + self.source = mock_source + self.description = "No description" + self.homepage = "https://example.com" + self.license = License.UNKNOWN + + return MockVimPlugin() + + +def test_vim_plugin_nix_expression(mock_plugin): + assert ( + mock_plugin.get_nix_expression() + == 'test = buildVimPluginFrom2Nix { pname = "test"; version = "1.0.0"; src = src; meta = with lib; { description = "No description"; homepage = "https://example.com"; license = with licenses; [ ]; }; };' + ) + + +class MockResponse: + def __init__(self, status_code: int, content: bytes): + self.status_code = status_code + self.content = content + + def json(self): + return json.loads(self.content) + + +def mock_request_get(repsonses: dict[str, MockResponse]): + respones_not_found = MockResponse(404, b'{"message": "Not Found"}') + + def mock_get(url: str, *args, **kwargs): + return repsonses.get(url, respones_not_found) + + return mock_get + + +@pytest.fixture() +def github_commits_response(): + return MockResponse( + 200, + json.dumps( + { + "sha": "6dcb09b5b57875f334f61aebed695e2e4193db5e", + "commit": { + "committer": { + "date": "2011-04-14T16:00:49Z", + }, + }, + } + ), + ) + + +@pytest.fixture() +def github_get(github_commits_response: MockResponse): + repos_response = MockResponse( + 200, + json.dumps( + { + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": False, + "default_branch": "master", + "license": { + "spdx_id": "MIT", + }, + } + ), + ) + responses = { + "https://api.github.com/repos/octocat/Hello-World": repos_response, + "https://api.github.com/repos/octocat/Hello-World/commits/master": github_commits_response, + } + return mock_request_get(responses) + + +@pytest.fixture() +def github_get_no_license(github_commits_response: MockResponse): + repos_response = MockResponse( + 200, + json.dumps( + { + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": False, + "default_branch": "master", + } + ), + ) + responses = { + "https://api.github.com/repos/octocat/Hello-World": repos_response, + "https://api.github.com/repos/octocat/Hello-World/commits/master": github_commits_response, + } + return mock_request_get(responses) + + +def test_github_plugin(mocker: MockFixture, github_get: Callable, url_source: UrlSource): + mocker.patch("requests.get", github_get) + url_source = mocker.patch("update_vim_plugins.nix.UrlSource", url_source) + + spec = PluginSpec.from_spec("octocat/Hello-World") + plugin = GitHubPlugin(spec) + + assert plugin.name == "Hello-World" + assert plugin.version == "2011-04-14" + assert plugin.description == "This your first repo!" + assert plugin.homepage == "https://github.com/octocat/Hello-World" + assert plugin.license == License.MIT + + +def test_github_plugin_no_license(mocker: MockFixture, github_get_no_license: Callable, url_source: UrlSource): + mocker.patch("requests.get", github_get_no_license) + url_source = mocker.patch("update_vim_plugins.nix.UrlSource", url_source) + + spec = PluginSpec.from_spec("octocat/Hello-World") + plugin = GitHubPlugin(spec) + + assert plugin.license == License.UNKNOWN diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_spec.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_spec.py new file mode 100644 index 00000000..2b9a1d24 --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/tests/test_spec.py @@ -0,0 +1,136 @@ +import pytest + +from update_vim_plugins.spec import PluginSpec, RepositoryHost + + +@pytest.fixture() +def owner(): + return "owner" + + +@pytest.fixture() +def repo(): + return "repo.nvim" + + +@pytest.fixture() +def branch(): + return "main" + + +@pytest.fixture() +def name(): + return "repo-nvim" + + +@pytest.fixture() +def license(): + return "mit" + + +def test_from_spec_simple(owner: str, repo: str): + vim_plugin = PluginSpec.from_spec(f"{owner}/{repo}") + + assert vim_plugin.owner == owner + assert vim_plugin.repo == repo + + +def test_from_spec_with_gitref(owner: str, repo: str, branch: str): + vim_plugin = PluginSpec.from_spec(f"{owner}/{repo}:{branch}") + + assert vim_plugin.branch == branch + + +def test_from_spec_with_name(owner: str, repo: str, name: str): + vim_plugin = PluginSpec.from_spec(f"{owner}/{repo}::{name}") + + assert vim_plugin.name == name + + +@pytest.mark.parametrize("host", RepositoryHost) +def test_from_spec_with_repository_host(owner: str, repo: str, host: RepositoryHost): + vim_plugin = PluginSpec.from_spec(f"{host.value}:{owner}/{repo}") + + assert vim_plugin.repository_host == host + + +def test_from_spec_without_repository_host(owner: str, repo: str): + vim_plugin = PluginSpec.from_spec(f"{owner}/{repo}") + + assert vim_plugin.repository_host == RepositoryHost.GITHUB + + +def test_from_spec_complex(owner: str, repo: str, branch: str, name: str): + vim_plugin = PluginSpec.from_spec(f"gitlab:{owner}/{repo}:{branch}:{name}") + + assert vim_plugin.repository_host == RepositoryHost.GITLAB + assert vim_plugin.owner == owner + assert vim_plugin.repo == repo + assert vim_plugin.branch == branch + assert vim_plugin.name == name + + +def test_from_spec_invalid_spec(): + with pytest.raises(ValueError): + PluginSpec.from_spec("invalid_spec") + + +def test_to_spec_simple(owner: str, repo: str): + vim_plugin = PluginSpec(RepositoryHost.GITHUB, owner, repo) + + assert vim_plugin.to_spec() == f"{owner}/{repo}" + + +def test_to_spec_with_branch(owner: str, repo: str, branch: str): + vim_plugin = PluginSpec(RepositoryHost.GITHUB, owner, repo, branch=branch) + assert vim_plugin.to_spec() == f"{owner}/{repo}:{branch}" + + +def test_to_spec_with_name(owner: str, repo: str, name: str): + vim_plugin = PluginSpec(RepositoryHost.GITHUB, owner, repo, name=name) + + assert vim_plugin.to_spec() == f"{owner}/{repo}::{name}" + + +@pytest.mark.parametrize("host", [RepositoryHost.GITLAB, RepositoryHost.SOURCEHUT]) +def test_to_spec_with_repository_host(host: RepositoryHost, owner: str, repo: str): + vim_plugin = PluginSpec(host, owner, repo) + + assert vim_plugin.to_spec() == f"{host.value}:{owner}/{repo}" + + +def test_to_spec_complex(owner: str, repo: str, branch: str, name: str, license: str): + vim_plugin = PluginSpec(RepositoryHost.GITLAB, owner, repo, branch=branch, name=name, license=license) + + assert vim_plugin.to_spec() == f"gitlab:{owner}/{repo}:{branch}:{name}:{license}" + + +def test_spec_equal(owner: str, repo: str): + vim_plugin = PluginSpec(RepositoryHost.GITHUB, owner, repo) + vim_plugin2 = PluginSpec(RepositoryHost.GITHUB, owner, repo) + + assert vim_plugin == vim_plugin2 + + +def test_spec_not_equal_different_branch(owner: str, repo: str): + vim_plugin = PluginSpec(RepositoryHost.GITHUB, owner, repo) + vim_plugin2 = PluginSpec(RepositoryHost.GITHUB, owner, repo, branch="main") + + assert vim_plugin != vim_plugin2 + + +def test_spec_not_equal_different_name(owner: str, repo: str): + vim_plugin = PluginSpec(RepositoryHost.GITHUB, owner, repo) + vim_plugin2 = PluginSpec(RepositoryHost.GITHUB, owner, repo, name="renamed") + + assert vim_plugin != vim_plugin2 + + +def test_spec_equal_same_normalized_name(owner: str): + repo = "repo.nvim" + name = "repo-nvim" + + vim_plugin = PluginSpec(RepositoryHost.GITHUB, owner, repo) + vim_plugin2 = PluginSpec(RepositoryHost.GITHUB, owner, repo, name=name) + + assert vim_plugin == vim_plugin2 diff --git a/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/update.py b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/update.py new file mode 100644 index 00000000..7eb3eeb4 --- /dev/null +++ b/pkgs/by-name/up/update-vim-plugins/update_vim_plugins/update.py @@ -0,0 +1,212 @@ +import subprocess +from random import shuffle +from cleo.helpers import argument, option +from cleo.commands.command import Command +from concurrent.futures import ThreadPoolExecutor, as_completed + +from pprint import pprint + +from .plugin import plugin_from_spec + +from .helpers import read_manifest_to_spec, get_const +from .helpers import JSON_FILE, PLUGINS_LIST_FILE, PKGS_FILE + +import json +import jsonpickle + +jsonpickle.set_encoder_options("json", sort_keys=True) + + +class UpdateCommand(Command): + name = "update" + description = "Generate nix module from input file" + arguments = [argument("plug_dir", description="Path to the plugin directory", optional=False)] + options = [ + option("all", "a", description="Update all plugins. Else only update new plugins", flag=True), + option("dry-run", "d", description="Show which plugins would be updated", flag=True), + ] + + def handle(self): + """Main command function""" + + plug_dir = self.argument("plug_dir") + self.specs = read_manifest_to_spec(plug_dir) + + if self.option("all"): + # update all plugins + spec_list = self.specs + known_plugins = [] + else: + # filter plugins we already know + spec_list = self.specs + + with open(get_const(JSON_FILE, plug_dir), "r") as json_file: + data = json.load(json_file) + + known_specs = list(filter(lambda x: x.line in data, spec_list)) + known_plugins = [jsonpickle.decode(data[x.line]) for x in known_specs] + + spec_list = list(filter(lambda x: x.line not in data, spec_list)) + + if self.option("dry-run"): + self.line("<comment>These plugins would be updated</comment>") + pprint(spec_list) + self.line(f"<info>Total:</info> {len(spec_list)}") + exit(0) + + processed_plugins, failed_plugins, failed_but_known = self.process_manifest(spec_list, plug_dir) + + processed_plugins += known_plugins # add plugins from .plugins.json + processed_plugins: list = sorted(set(processed_plugins)) # remove duplicates based only on source line + + self.check_duplicates(processed_plugins) + + if failed_plugins != []: + self.line("<error>Not processed:</error> The following plugins could not be updated") + for s, e in failed_plugins: + self.line(f" - {s!r} - {e}") + + if failed_but_known != []: + self.line( + "<error>Not updated:</error> The following plugins could not be updated but an older version is known" + ) + for s, e in failed_but_known: + self.line(f" - {s!r} - {e}") + + # update plugin "database" + self.write_plugins_json(processed_plugins, plug_dir) + + # generate output + self.write_plugins_nix(processed_plugins, plug_dir) + + self.write_plugins_markdown(processed_plugins, plug_dir) + + self.line("<comment>Done</comment>") + + def write_plugins_markdown(self, plugins, plug_dir): + """Write the list of all plugins to PLUGINS_LIST_FILE in markdown""" + + plugins.sort() + + self.line("<info>Updating plugins.md</info>") + + header = f" - Plugin count: {len(plugins)}\n\n| Repo | Last Update | Nix package name | Last checked |\n|:---|:---|:---|:---|\n" + + with open(get_const(PLUGINS_LIST_FILE, plug_dir), "w") as file: + file.write(header) + for plugin in plugins: + file.write(f"{plugin.to_markdown()}\n") + + def write_plugins_nix(self, plugins, plug_dir): + self.line("<info>Generating nix output</info>") + + plugins.sort() + + header = "{ lib, buildVimPlugin, fetchurl, fetchgit }: {" + footer = "}" + + with open(get_const(PKGS_FILE, plug_dir), "w") as file: + file.write(header) + for plugin in plugins: + file.write(f"{plugin.to_nix()}\n") + file.write(footer) + + self.line("<info>Formatting nix output</info>") + + subprocess.run( + ["alejandra", get_const(PKGS_FILE, plug_dir)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + def write_plugins_json(self, plugins, plug_dir): + self.line("<info>Storing results in .plugins.json</info>") + + plugins.sort() + + with open(get_const(JSON_FILE, plug_dir), "r+") as json_file: + data = json.load(json_file) + + for plugin in plugins: + data.update({f"{plugin.source_line}": plugin.to_json()}) + + json_file.seek(0) + json_file.write(json.dumps(data, indent=2, sort_keys=True)) + json_file.truncate() + + def check_duplicates(self, plugins): + """check for duplicates in proccesed_plugins""" + error = False + for i, plugin in enumerate(plugins): + for p in plugins[i + 1 :]: + if plugin.name == p.name: + self.line( + f"<error>Error:</error> The following two lines produce the same plugin name:\n - {plugin.source_line}\n - {p.source_line}\n -> {p.name}" + ) + error = True + + # We want to exit if the resulting nix file would be broken + # But we want to go through all plugins before we do so + if error: + exit(1) + + def generate_plugin(self, spec, i, size, plug_dir): + debug_string = "" + + processed_plugin = None + failed_but_known = None + failed_plugin = None + try: + debug_string += f" - <info>({i+1}/{size}) Processing</info> {spec!r}\n" + vim_plugin = plugin_from_spec(spec) + debug_string += f" • <comment>Success</comment> {vim_plugin!r}\n" + processed_plugin = vim_plugin + except Exception as e: + debug_string += f" • <error>Error:</error> Could not update <info>{spec.name}</info>. Keeping old values. Reason: {e}\n" + with open(get_const(JSON_FILE, plug_dir), "r") as json_file: + data = json.load(json_file) + + plugin_json = data.get(spec.line) + if plugin_json: + vim_plugin = jsonpickle.decode(plugin_json) + processed_plugin = vim_plugin + failed_but_known = (vim_plugin, e) + else: + debug_string += f" • <error>Error:</error> No entries for <info>{spec.name}</info> in '.plugins.json'. Skipping...\n" + failed_plugin = (spec, e) + + self.line(debug_string.strip()) + + return processed_plugin, failed_plugin, failed_but_known + + def process_manifest(self, spec_list, plug_dir): + """Read specs in 'spec_list' and generate plugins""" + + size = len(spec_list) + + # We have to assume that we will reach an api limit. Therefore + # we randomize the spec list to give every entry the same change to be updated and + # not favor those at the start of the list + shuffle(spec_list) + + with ThreadPoolExecutor() as executor: + futures = [ + executor.submit(self.generate_plugin, spec, i, size, plug_dir) for i, spec in enumerate(spec_list) + ] + results = [future.result() for future in as_completed(futures)] + + processed_plugins = [r[0] for r in results] + failed_plugins = [r[1] for r in results] + failed_but_known = [r[2] for r in results] + + processed_plugins = list(filter(lambda x: x is not None, processed_plugins)) + failed_plugins = list(filter(lambda x: x is not None, failed_plugins)) + failed_but_known = list(filter(lambda x: x is not None, failed_but_known)) + + processed_plugins.sort() + failed_plugins.sort() + failed_but_known.sort() + + assert len(processed_plugins) == len(spec_list) - len(failed_plugins) + + return processed_plugins, failed_plugins, failed_but_known |