From 03461b80bf60e9bc740f5fccd9a2f3329a23fcf2 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 16:21:03 +0100 Subject: [PATCH] switch to strict typing with pyright Signed-off-by: Ivan Schaller --- .woodpecker/tests.yml | 7 ++++++ contrib/requirements_dev.txt | 1 + justfile | 8 ++++++- mangadlp/api/mangadex.py | 46 ++++++++++++++++++++---------------- mangadlp/app.py | 20 ++++++++-------- mangadlp/cache.py | 12 ++++++---- mangadlp/cli.py | 21 ++++++++-------- mangadlp/downloader.py | 8 +++---- mangadlp/hooks.py | 3 ++- mangadlp/logger.py | 9 +++---- mangadlp/metadata.py | 12 ++++++---- mangadlp/utils.py | 6 ++--- pyproject.toml | 17 +++++++++++-- 13 files changed, 104 insertions(+), 66 deletions(-) diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml index bb8ddc6..06097c7 100644 --- a/.woodpecker/tests.yml +++ b/.woodpecker/tests.yml @@ -33,6 +33,13 @@ pipeline: commands: - just test_mypy + # check static typing - python + test-pyright: + image: cr.44net.ch/ci-plugins/tests + pull: true + commands: + - just test_pyright + # ruff test - python test-ruff: image: cr.44net.ch/ci-plugins/tests diff --git a/contrib/requirements_dev.txt b/contrib/requirements_dev.txt index b8d5219..680db66 100644 --- a/contrib/requirements_dev.txt +++ b/contrib/requirements_dev.txt @@ -17,3 +17,4 @@ black>=22.1.0 mypy>=0.940 tox>=3.24.5 ruff>=0.0.247 +pyright>=1.1.294 diff --git a/justfile b/justfile index 1fbbd5f..d2e0b38 100755 --- a/justfile +++ b/justfile @@ -82,7 +82,10 @@ test_black: @python3 -m black --check --diff . test_mypy: - @python3 -m mypy --install-types --non-interactive --ignore-missing-imports . + @python3 -m mypy --install-types --non-interactive --ignore-missing-imports mangadlp/ + +test_pyright: + @python3 -m pyright mangadlp/ test_ruff: @python3 -m ruff --diff . @@ -118,6 +121,7 @@ lint: just test_shfmt just test_black just test_mypy + just test_pyright just test_ruff @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" @@ -127,6 +131,7 @@ tests: just test_shfmt just test_black just test_mypy + just test_pyright just test_ruff just test_pytest @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" @@ -137,6 +142,7 @@ tests_full: just test_shfmt just test_black just test_mypy + just test_pyright just test_ruff just test_build just test_tox diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index a6e3f11..5b79460 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -1,5 +1,6 @@ import re from time import sleep +from typing import Any, Dict, List, Union import requests from loguru import logger as log @@ -65,10 +66,10 @@ class Mangadex: log.error("No valid UUID found") raise exc - return uuid + return uuid # pyright:ignore # make initial request - def get_manga_data(self) -> dict: + def get_manga_data(self) -> Dict[str, Any]: log.debug(f"Getting manga data for: {self.manga_uuid}") counter = 1 while counter <= 3: @@ -85,12 +86,14 @@ class Mangadex: counter += 1 else: break + + response_body: Dict[str, Dict[str, Any]] = response.json() # pyright:ignore # check if manga exists - if response.json()["result"] != "ok": + if response_body["result"] != "ok": # type:ignore log.error("Manga not found") raise KeyError - return response.json()["data"] + return response_body["data"] # get the title of the manga (and fix the filename) def get_manga_title(self) -> str: @@ -112,7 +115,7 @@ class Mangadex: if item.get(self.language): alt_title = item break - title = alt_title[self.language] + title = alt_title[self.language] # pyright:ignore except (KeyError, UnboundLocalError): log.warning( "Manga title also not found in alt titles. Falling back to english title" @@ -133,7 +136,7 @@ class Mangadex: timeout=10, ) try: - total_chapters = r.json()["total"] + total_chapters: int = r.json()["total"] except Exception as exc: log.error( "Error retrieving the chapters list. Did you specify a valid language code?" @@ -147,7 +150,7 @@ class Mangadex: return total_chapters # get chapter data like name, uuid etc - def get_chapter_data(self) -> dict: + def get_chapter_data(self) -> Dict[str, Dict[str, Union[str, int]]]: log.debug(f"Getting chapter data for: {self.manga_uuid}") api_sorting = "order[chapter]=asc&order[volume]=asc" # check for chapters in specified lang @@ -161,8 +164,9 @@ class Mangadex: f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}", timeout=10, ) - for chapter in r.json()["data"]: - attributes: dict = chapter["attributes"] + response_body: Dict[str, Any] = r.json() + for chapter in response_body["data"]: + attributes: Dict[str, Any] = chapter["attributes"] # chapter infos from feed chapter_num: str = attributes.get("chapter") or "" chapter_vol: str = attributes.get("volume") or "" @@ -201,10 +205,10 @@ class Mangadex: # increase offset for mangas with more than 500 chapters offset += 500 - return chapter_data + return chapter_data # type:ignore # get images for the chapter (mangadex@home) - def get_chapter_images(self, chapter: str, wait_time: float) -> list: + def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]: log.debug(f"Getting chapter images for: {self.manga_uuid}") athome_url = f"{self.api_base_url}/at-home/server" chapter_uuid = self.manga_chapter_data[chapter]["uuid"] @@ -238,11 +242,11 @@ class Mangadex: if api_error: return [] - chapter_hash = api_data["chapter"]["hash"] - chapter_img_data = api_data["chapter"]["data"] + chapter_hash = api_data["chapter"]["hash"] # pyright:ignore + chapter_img_data = api_data["chapter"]["data"] # pyright:ignore # get list of image urls - image_urls = [] + image_urls: List[str] = [] for image in chapter_img_data: image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}") @@ -251,12 +255,12 @@ class Mangadex: return image_urls # create list of chapters - def create_chapter_list(self) -> list: + def create_chapter_list(self) -> List[str]: log.debug(f"Creating chapter list for: {self.manga_uuid}") - chapter_list = [] + chapter_list: List[str] = [] for data in self.manga_chapter_data.values(): - chapter_number: str = data["chapter"] - volume_number: str = data["volume"] + chapter_number: str = data["chapter"] # type:ignore + volume_number: str = data["volume"] # type:ignore if self.forcevol: chapter_list.append(f"{volume_number}:{chapter_number}") else: @@ -264,12 +268,12 @@ class Mangadex: return chapter_list - def create_metadata(self, chapter: str) -> dict: + def create_metadata(self, chapter: str) -> Dict[str, Union[str, int, None]]: log.info("Creating metadata from api") chapter_data = self.manga_chapter_data[chapter] try: - volume = int(chapter_data.get("volume")) + volume = int(chapter_data["volume"]) except (ValueError, TypeError): volume = None metadata = { @@ -285,4 +289,4 @@ class Mangadex: "Web": f"https://mangadex.org/title/{self.manga_uuid}", } - return metadata + return metadata # pyright:ignore diff --git a/mangadlp/app.py b/mangadlp/app.py index 439deb3..4b8ad91 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -1,7 +1,7 @@ import re import shutil from pathlib import Path -from typing import Any, Union +from typing import Any, Dict, List, Tuple, Union from loguru import logger as log @@ -23,7 +23,7 @@ def match_api(url_uuid: str) -> type: The class of the API to use """ # apis to check - apis: list[tuple[str, re.Pattern, type]] = [ + apis: List[Tuple[str, re.Pattern[str], type]] = [ ( "mangadex.org", re.compile( @@ -108,7 +108,7 @@ class MangaDLP: self.chapter_post_hook_cmd = chapter_post_hook_cmd self.cache_path = cache_path self.add_metadata = add_metadata - self.hook_infos: dict = {} + self.hook_infos: Dict[str, Any] = {} # prepare everything self._prepare() @@ -226,7 +226,7 @@ class MangaDLP: skipped_chapters: list[Any] = [] error_chapters: list[Any] = [] for chapter in chapters_to_download: - if self.cache_path and chapter in cached_chapters: + if self.cache_path and chapter in cached_chapters: # pyright:ignore log.info(f"Chapter '{chapter}' is in cache. Skipping download") continue @@ -240,7 +240,7 @@ class MangaDLP: skipped_chapters.append(chapter) # update cache if self.cache_path: - cache.add_chapter(chapter) + cache.add_chapter(chapter) # pyright:ignore continue except Exception: # skip download/packing due to an error @@ -273,7 +273,7 @@ class MangaDLP: # update cache if self.cache_path: - cache.add_chapter(chapter) + cache.add_chapter(chapter) # pyright:ignore # start chapter post hook run_hook( @@ -310,7 +310,7 @@ class MangaDLP: # once called per chapter def get_chapter(self, chapter: str) -> Path: # get chapter infos - chapter_infos: dict = self.api.manga_chapter_data[chapter] + chapter_infos: Dict[str, Union[str, int]] = self.api.manga_chapter_data[chapter] log.debug(f"Chapter infos: {chapter_infos}") # get image urls for chapter @@ -342,8 +342,8 @@ class MangaDLP: # get filename for chapter (without suffix) chapter_filename = utils.get_filename( self.manga_title, - chapter_infos["name"], - chapter_infos["volume"], + chapter_infos["name"], # type:ignore + chapter_infos["volume"], # type:ignore chapter, self.forcevol, self.name_format, @@ -352,7 +352,7 @@ class MangaDLP: log.debug(f"Filename: '{chapter_filename}'") # set download path for chapter (image folder) - chapter_path = self.manga_path / chapter_filename + chapter_path: Path = self.manga_path / chapter_filename # set archive path with file format chapter_archive_path = Path(f"{chapter_path}{self.file_format}") diff --git a/mangadlp/cache.py b/mangadlp/cache.py index a2077b5..1781dba 100644 --- a/mangadlp/cache.py +++ b/mangadlp/cache.py @@ -26,12 +26,14 @@ class CacheDB: if not self.db_data.get(self.db_key): self.db_data[self.db_key] = {} - self.db_uuid_data: dict = self.db_data[self.db_key] + self.db_uuid_data = self.db_data[self.db_key] if not self.db_uuid_data.get("name"): self.db_uuid_data.update({"name": self.name}) self._write_db() - self.db_uuid_chapters: list = self.db_uuid_data.get("chapters") or [] + self.db_uuid_chapters: List[str] = ( + self.db_uuid_data.get("chapters") or [] # type:ignore + ) def _prepare_db(self) -> None: if self.db_path.exists(): @@ -44,11 +46,11 @@ class CacheDB: log.error("Can't create db-file") raise exc - def _read_db(self) -> Dict[str, dict]: + def _read_db(self) -> Dict[str, Dict[str, Union[str, List[str]]]]: log.info(f"Reading cache-db: {self.db_path}") try: db_txt = self.db_path.read_text(encoding="utf8") - db_dict: dict[str, dict] = json.loads(db_txt) + db_dict: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(db_txt) except Exception as exc: log.error("Can't load cache-db") raise exc @@ -73,7 +75,7 @@ class CacheDB: raise exc -def sort_chapters(chapters: list) -> List[str]: +def sort_chapters(chapters: List[str]) -> List[str]: try: sorted_list = sorted(chapters, key=float) except Exception: diff --git a/mangadlp/cli.py b/mangadlp/cli.py index 0da36a2..998e966 100644 --- a/mangadlp/cli.py +++ b/mangadlp/cli.py @@ -1,5 +1,6 @@ import sys from pathlib import Path +from typing import Any, List import click from click_option_group import ( @@ -15,7 +16,7 @@ from mangadlp.logger import prepare_logger # read in the list of links from a file -def readin_list(_ctx, _param, value) -> list: +def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: if not value: return [] @@ -38,8 +39,8 @@ def readin_list(_ctx, _param, value) -> list: @click.help_option() @click.version_option(version=__version__, package_name="manga-dlp") # manga selection -@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup) -@optgroup.option( +@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup) # type: ignore +@optgroup.option( # type: ignore "-u", "--url", "--uuid", @@ -49,19 +50,19 @@ def readin_list(_ctx, _param, value) -> list: show_default=True, help="URL or UUID of the manga", ) -@optgroup.option( +@optgroup.option( # type: ignore "--read", "read_mangas", is_eager=True, callback=readin_list, - type=click.Path(exists=True, dir_okay=False), + type=click.Path(exists=True, dir_okay=False, path_type=str), default=None, show_default=True, help="Path of file with manga links to download. One per line", ) # logging options -@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup) -@optgroup.option( +@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup) # type: ignore +@optgroup.option( # type: ignore "--loglevel", "verbosity", type=int, @@ -69,7 +70,7 @@ def readin_list(_ctx, _param, value) -> list: show_default=True, help="Custom log level", ) -@optgroup.option( +@optgroup.option( # type: ignore "--warn", "verbosity", flag_value=30, @@ -77,7 +78,7 @@ def readin_list(_ctx, _param, value) -> list: show_default=True, help="Only log warnings and higher", ) -@optgroup.option( +@optgroup.option( # type: ignore "--debug", "verbosity", flag_value=10, @@ -227,7 +228,7 @@ def readin_list(_ctx, _param, value) -> list: help="Enable/disable creation of metadata via ComicInfo.xml", ) @click.pass_context -def main(ctx: click.Context, **kwargs) -> None: +def main(ctx: click.Context, **kwargs: Any) -> None: """Script to download mangas from various sites.""" url_uuid: str = kwargs.pop("url_uuid") read_mangas: list[str] = kwargs.pop("read_mangas") diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index 3826531..4630dd6 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -2,7 +2,7 @@ import logging import shutil from pathlib import Path from time import sleep -from typing import Union +from typing import List, Union import requests from loguru import logger as log @@ -12,7 +12,7 @@ from mangadlp import utils # download images def download_chapter( - image_urls: list, + image_urls: List[str], chapter_path: Union[str, Path], download_wait: float, ) -> None: @@ -48,8 +48,8 @@ def download_chapter( # write image try: with image_path.open("wb") as file: - r.raw.decode_content = True - shutil.copyfileobj(r.raw, file) + r.raw.decode_content = True # pyright:ignore + shutil.copyfileobj(r.raw, file) # pyright:ignore except Exception as exc: log.error("Can't write file") raise exc diff --git a/mangadlp/hooks.py b/mangadlp/hooks.py index 514af64..db3d50e 100644 --- a/mangadlp/hooks.py +++ b/mangadlp/hooks.py @@ -1,10 +1,11 @@ import os import subprocess +from typing import Any from loguru import logger as log -def run_hook(command: str, hook_type: str, **kwargs) -> int: +def run_hook(command: str, hook_type: str, **kwargs: Any) -> int: """Run a command. Run a command with subprocess.run and add kwargs to the environment. diff --git a/mangadlp/logger.py b/mangadlp/logger.py index f332ed4..f0b0897 100644 --- a/mangadlp/logger.py +++ b/mangadlp/logger.py @@ -1,5 +1,6 @@ import logging import sys +from typing import Any, Dict from loguru import logger @@ -10,7 +11,7 @@ LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | [{level: <7}] [{name: class InterceptHandler(logging.Handler): """Intercept python logging messages and log them via loguru.logger.""" - def emit(self, record): + def emit(self, record: Any) -> None: # Get corresponding Loguru level if it exists try: level = logger.level(record.levelname).name @@ -19,8 +20,8 @@ class InterceptHandler(logging.Handler): # Find caller from where originated the logged message frame, depth = logging.currentframe(), 2 - while frame.f_code.co_filename == logging.__file__: - frame = frame.f_back + while frame.f_code.co_filename == logging.__file__: # pyright:ignore + frame = frame.f_back # type: ignore depth += 1 logger.opt(depth=depth, exception=record.exc_info).log( @@ -30,7 +31,7 @@ class InterceptHandler(logging.Handler): # init logger with format and log level def prepare_logger(loglevel: int = 20) -> None: - config: dict = { + config: Dict[str, Any] = { "handlers": [ { "sink": sys.stdout, diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index 3e31463..9ee1478 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, Tuple +from typing import Any, Dict, Tuple, Union import xmltodict from loguru import logger as log @@ -8,7 +8,7 @@ METADATA_FILENAME = "ComicInfo.xml" METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml") # define metadata types, defaults and valid values. an empty list means no value check # {key: (type, default value, valid values)} -METADATA_TYPES: Dict[str, Tuple[type, Any, list]] = { +METADATA_TYPES: Dict[str, Tuple[type, Any, list[Union[str, int]]]] = { "Title": (str, None, []), "Series": (str, None, []), "Number": (str, None, []), @@ -59,10 +59,12 @@ METADATA_TYPES: Dict[str, Tuple[type, Any, list]] = { } -def validate_metadata(metadata_in: dict) -> Dict[str, dict]: +def validate_metadata( + metadata_in: Dict[str, Union[str, int]] +) -> Dict[str, Dict[str, Union[str, int]]]: log.info("Validating metadata") - metadata_valid: dict[str, dict] = {"ComicInfo": {}} + metadata_valid: dict[str, Dict[str, Union[str, int]]] = {"ComicInfo": {}} for key, value in METADATA_TYPES.items(): metadata_type, metadata_default, metadata_validation = value @@ -104,7 +106,7 @@ def validate_metadata(metadata_in: dict) -> Dict[str, dict]: return metadata_valid -def write_metadata(chapter_path: Path, metadata: dict) -> None: +def write_metadata(chapter_path: Path, metadata: Dict[str, Union[str, int]]) -> None: if metadata["Format"] == "pdf": log.warning("Can't add metadata for pdf format. Skipping") return diff --git a/mangadlp/utils.py b/mangadlp/utils.py index d305a45..0d3c31e 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -24,7 +24,7 @@ def make_archive(chapter_path: Path, file_format: str) -> None: def make_pdf(chapter_path: Path) -> None: try: - import img2pdf # pylint: disable=import-outside-toplevel + import img2pdf # pylint: disable=import-outside-toplevel # pyright:ignore except Exception as exc: log.error("Cant import img2pdf. Please install it first") raise exc @@ -34,14 +34,14 @@ def make_pdf(chapter_path: Path) -> None: for file in chapter_path.iterdir(): images.append(str(file)) try: - pdf_path.write_bytes(img2pdf.convert(images)) + pdf_path.write_bytes(img2pdf.convert(images)) # pyright:ignore except Exception as exc: log.error("Can't create '.pdf' archive") raise exc # create a list of chapters -def get_chapter_list(chapters: str, available_chapters: list) -> List[str]: +def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]: # check if there are available chapter chapter_list: list[str] = [] for chapter in chapters.split(","): diff --git a/pyproject.toml b/pyproject.toml index 49f557a..ac85e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ # mypy [tool.mypy] -#strict = true +strict = true python_version = "3.9" disallow_untyped_defs = false follow_imports = "normal" @@ -84,6 +84,18 @@ show_error_codes = true pretty = true no_implicit_optional = false +# pyright + +[tool.pyright] +typeCheckingMode = "strict" +pythonVersion = "3.9" +reportUnnecessaryTypeIgnoreComment = true +reportShadowedImports = true +reportUnusedExpression = true +reportMatchNotExhaustive = true +# venvPath = "." +# venv = "venv" + # ruff [tool.ruff] @@ -103,10 +115,11 @@ select = [ ] line-length = 88 fix = true +show-fixes = true format = "grouped" ignore-init-module-imports = true respect-gitignore = true -ignore = ["E501", "D103", "D100"] +ignore = ["E501", "D103", "D100", "D102", "PLR2004"] exclude = [ ".direnv", ".git",