add typed dicts for type hinting
All checks were successful
ci/woodpecker/push/tests Pipeline was successful

This commit is contained in:
Ivan Schaller 2023-02-20 14:03:40 +01:00
parent e2f0a8b41f
commit bde2b9ebe9
5 changed files with 76 additions and 32 deletions

View file

@ -1,4 +1,6 @@
from typing import List, Dict, Union from typing import Dict, List, Union
from mangadlp.metadata import ChapterData
# api template for manga-dlp # api template for manga-dlp
@ -37,20 +39,20 @@ class YourAPI:
self.manga_uuid = "abc" self.manga_uuid = "abc"
self.manga_title = "abc" self.manga_title = "abc"
self.chapter_list = ["1", "2", "2.1", "5", "10"] self.chapter_list = ["1", "2", "2.1", "5", "10"]
self.manga_chapter_data: Dict[ self.manga_chapter_data: Dict[str, ChapterData] = { # example data
str, Dict[str, Union[str, int]]
] = { # example data
"1": { "1": {
"uuid": "abc", "uuid": "abc",
"volume": "1", "volume": "1",
"chapter": "1", "chapter": "1",
"name": "test", "name": "test",
"pages" 2,
}, },
"2": { "2": {
"uuid": "abc", "uuid": "abc",
"volume": "1", "volume": "1",
"chapter": "2", "chapter": "2",
"name": "test", "name": "test",
"pages": 45,
}, },
} }
# or with --forcevol # or with --forcevol
@ -86,7 +88,7 @@ class YourAPI:
"https://abc.def/image/12345.png", "https://abc.def/image/12345.png",
] ]
def create_metadata(self, chapter: str) -> Dict[str, Union[str, int, None]]: def create_metadata(self, chapter: str) -> ComicInfo:
"""Get metadata with correct keys for ComicInfo.xml. """Get metadata with correct keys for ComicInfo.xml.
Provide as much metadata as possible. empty/false values will be ignored. Provide as much metadata as possible. empty/false values will be ignored.

View file

@ -81,9 +81,6 @@ test_shfmt:
test_black: test_black:
@python3 -m black --check --diff mangadlp/ @python3 -m black --check --diff mangadlp/
test_mypy:
@python3 -m mypy --install-types --non-interactive --ignore-missing-imports mangadlp/
test_pyright: test_pyright:
@python3 -m pyright mangadlp/ @python3 -m pyright mangadlp/
@ -120,7 +117,6 @@ lint:
-just test_ci_conf -just test_ci_conf
just test_shfmt just test_shfmt
just test_black just test_black
just test_mypy
just test_pyright just test_pyright
just test_ruff just test_ruff
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
@ -130,7 +126,6 @@ tests:
-just test_ci_conf -just test_ci_conf
just test_shfmt just test_shfmt
just test_black just test_black
just test_mypy
just test_pyright just test_pyright
just test_ruff just test_ruff
just test_pytest just test_pytest
@ -141,7 +136,6 @@ tests_full:
-just test_ci_conf -just test_ci_conf
just test_shfmt just test_shfmt
just test_black just test_black
just test_mypy
just test_pyright just test_pyright
just test_ruff just test_ruff
just test_build just test_build

View file

@ -1,11 +1,12 @@
import re import re
from time import sleep from time import sleep
from typing import Any, Dict, List, Union from typing import Any, Dict, List
import requests import requests
from loguru import logger as log from loguru import logger as log
from mangadlp import utils from mangadlp import utils
from mangadlp.metadata import ChapterData, ComicInfo
class Mangadex: class Mangadex:
@ -150,13 +151,13 @@ class Mangadex:
return total_chapters return total_chapters
# get chapter data like name, uuid etc # get chapter data like name, uuid etc
def get_chapter_data(self) -> Dict[str, Dict[str, Union[str, int]]]: def get_chapter_data(self) -> Dict[str, ChapterData]:
log.debug(f"Getting chapter data for: {self.manga_uuid}") log.debug(f"Getting chapter data for: {self.manga_uuid}")
api_sorting = "order[chapter]=asc&order[volume]=asc" api_sorting = "order[chapter]=asc&order[volume]=asc"
# check for chapters in specified lang # check for chapters in specified lang
total_chapters = self.check_chapter_lang() total_chapters = self.check_chapter_lang()
chapter_data = {} chapter_data: dict[str, ChapterData] = {}
last_volume, last_chapter = ("", "") last_volume, last_chapter = ("", "")
offset = 0 offset = 0
while offset < total_chapters: # if more than 500 chapters while offset < total_chapters: # if more than 500 chapters
@ -205,7 +206,7 @@ class Mangadex:
# increase offset for mangas with more than 500 chapters # increase offset for mangas with more than 500 chapters
offset += 500 offset += 500
return chapter_data # type:ignore return chapter_data
# get images for the chapter (mangadex@home) # get images for the chapter (mangadex@home)
def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]: def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]:
@ -259,8 +260,8 @@ class Mangadex:
log.debug(f"Creating chapter list for: {self.manga_uuid}") log.debug(f"Creating chapter list for: {self.manga_uuid}")
chapter_list: List[str] = [] chapter_list: List[str] = []
for data in self.manga_chapter_data.values(): for data in self.manga_chapter_data.values():
chapter_number: str = data["chapter"] # type:ignore chapter_number: str = data["chapter"]
volume_number: str = data["volume"] # type:ignore volume_number: str = data["volume"]
if self.forcevol: if self.forcevol:
chapter_list.append(f"{volume_number}:{chapter_number}") chapter_list.append(f"{volume_number}:{chapter_number}")
else: else:
@ -268,7 +269,7 @@ class Mangadex:
return chapter_list return chapter_list
def create_metadata(self, chapter: str) -> Dict[str, Union[str, int, None]]: def create_metadata(self, chapter: str) -> ComicInfo:
log.info("Creating metadata from api") log.info("Creating metadata from api")
chapter_data = self.manga_chapter_data[chapter] chapter_data = self.manga_chapter_data[chapter]
@ -276,7 +277,7 @@ class Mangadex:
volume = int(chapter_data["volume"]) volume = int(chapter_data["volume"])
except (ValueError, TypeError): except (ValueError, TypeError):
volume = None volume = None
metadata = { metadata: ComicInfo = {
"Volume": volume, "Volume": volume,
"Number": chapter_data.get("chapter"), "Number": chapter_data.get("chapter"),
"PageCount": chapter_data.get("pages"), "PageCount": chapter_data.get("pages"),
@ -289,4 +290,4 @@ class Mangadex:
"Web": f"https://mangadex.org/title/{self.manga_uuid}", "Web": f"https://mangadex.org/title/{self.manga_uuid}",
} }
return metadata # pyright:ignore return metadata

View file

@ -9,7 +9,7 @@ from mangadlp import downloader, utils
from mangadlp.api.mangadex import Mangadex from mangadlp.api.mangadex import Mangadex
from mangadlp.cache import CacheDB from mangadlp.cache import CacheDB
from mangadlp.hooks import run_hook from mangadlp.hooks import run_hook
from mangadlp.metadata import write_metadata from mangadlp.metadata import ChapterData, write_metadata
from mangadlp.utils import get_file_format from mangadlp.utils import get_file_format
@ -310,7 +310,7 @@ class MangaDLP:
# once called per chapter # once called per chapter
def get_chapter(self, chapter: str) -> Path: def get_chapter(self, chapter: str) -> Path:
# get chapter infos # get chapter infos
chapter_infos: Dict[str, Union[str, int]] = self.api.manga_chapter_data[chapter] chapter_infos: ChapterData = self.api.manga_chapter_data[chapter]
log.debug(f"Chapter infos: {chapter_infos}") log.debug(f"Chapter infos: {chapter_infos}")
# get image urls for chapter # get image urls for chapter
@ -342,8 +342,8 @@ class MangaDLP:
# get filename for chapter (without suffix) # get filename for chapter (without suffix)
chapter_filename = utils.get_filename( chapter_filename = utils.get_filename(
self.manga_title, self.manga_title,
chapter_infos["name"], # type:ignore chapter_infos["name"],
chapter_infos["volume"], # type:ignore chapter_infos["volume"],
chapter, chapter,
self.forcevol, self.forcevol,
self.name_format, self.name_format,

View file

@ -1,5 +1,5 @@
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Tuple, Union from typing import Any, Dict, Optional, Tuple, TypedDict, Union
import xmltodict import xmltodict
from loguru import logger as log from loguru import logger as log
@ -59,12 +59,59 @@ METADATA_TYPES: Dict[str, Tuple[type, Any, list[Union[str, int]]]] = {
} }
def validate_metadata( class ComicInfo(TypedDict, total=False):
metadata_in: Dict[str, Union[str, int]] """ComicInfo.xml basic types.
) -> Dict[str, Dict[str, Union[str, int]]]:
Validation is done via metadata.validate_metadata()
All valid types and values are specified in metadata.METADATA_TYPES
"""
Title: Optional[str]
Series: Optional[str]
Number: Optional[str]
Count: Optional[int]
Volume: Optional[int]
AlternateSeries: Optional[str]
AlternateNumber: Optional[str]
AlternateCount: Optional[int]
Summary: Optional[str]
Notes: Optional[str]
Year: Optional[int]
Month: Optional[int]
Day: Optional[int]
Writer: Optional[str]
Colorist: Optional[str]
Publisher: Optional[str]
Genre: Optional[str]
Web: Optional[str]
PageCount: Optional[int]
LanguageISO: Optional[str]
Format: Optional[str]
BlackAndWhite: Optional[str]
Manga: Optional[str]
ScanInformation: Optional[str]
SeriesGroup: Optional[str]
AgeRating: Optional[str]
CommunityRating: Optional[int]
class ChapterData(TypedDict):
"""Basic chapter-data types.
All values have to be provided.
"""
uuid: str
volume: str
chapter: str
name: str
pages: int
def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]:
log.info("Validating metadata") log.info("Validating metadata")
metadata_valid: dict[str, Dict[str, Union[str, int]]] = {"ComicInfo": {}} metadata_valid: dict[str, ComicInfo] = {"ComicInfo": {}}
for key, value in METADATA_TYPES.items(): for key, value in METADATA_TYPES.items():
metadata_type, metadata_default, metadata_validation = value metadata_type, metadata_default, metadata_validation = value
@ -77,7 +124,7 @@ def validate_metadata(
# check if metadata key is available # check if metadata key is available
try: try:
md_to_check = metadata_in[key] md_to_check: Union[str, int, None] = metadata_in[key]
except KeyError: except KeyError:
continue continue
# check if provided metadata item is empty # check if provided metadata item is empty
@ -106,8 +153,8 @@ def validate_metadata(
return metadata_valid return metadata_valid
def write_metadata(chapter_path: Path, metadata: Dict[str, Union[str, int]]) -> None: def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None:
if metadata["Format"] == "pdf": if metadata["Format"] == "pdf": # pyright:ignore
log.warning("Can't add metadata for pdf format. Skipping") log.warning("Can't add metadata for pdf format. Skipping")
return return