diff --git a/CHANGELOG.md b/CHANGELOG.md index 30852bb..676caa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Added `xmltodict` as a package requirement - Cache now also saves the manga title - New tests +- More typo annotations for function, compatible with python3.8 ### Fixed diff --git a/contrib/api_template.py b/contrib/api_template.py index 63b544c..aa95174 100644 --- a/contrib/api_template.py +++ b/contrib/api_template.py @@ -1,5 +1,4 @@ # api template for manga-dlp -from typing import Any class YourAPI: diff --git a/mangadlp/app.py b/mangadlp/app.py index aff8da1..9f0c93c 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -17,18 +17,23 @@ class MangaDLP: After initialization, start the script with the function get_manga(). Args: - url_uuid (str): URL or UUID of the manga - language (str): Manga language with country codes. "en" --> english - chapters (str): Chapters to download, "all" for every chapter available - list_chapters (bool): List all available chapters and exit - file_format (str): Archive format to create. An empty string means don't archive the folder - forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume - download_path (str/Path): Download path. Defaults to '/downloads' - download_wait (float): Time to wait for each picture to download in seconds - + url_uuid: URL or UUID of the manga + language: Manga language with country codes. "en" --> english + chapters: Chapters to download, "all" for every chapter available + list_chapters: List all available chapters and exit + file_format: Archive format to create. An empty string means don't archive the folder + forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume + download_path: Download path. Defaults to '/downloads' + download_wait: Time to wait for each picture to download in seconds + manga_pre_hook_cmd: Command(s) to before after each manga + manga_post_hook_cmd: Command(s) to run after each manga + chapter_pre_hook_cmd: Command(s) to run before each chapter + chapter_post_hook_cmd: Command(s) to run after each chapter + cache_path: Path to the json cache. If emitted, no cache is used + add_metadata: Flag to toggle creation & inclusion of metadata """ - def __init__( + def __init__( # pylint: disable=too-many-locals self, url_uuid: str, language: str = "en", @@ -62,9 +67,9 @@ class MangaDLP: self.manga_post_hook_cmd = manga_post_hook_cmd self.chapter_pre_hook_cmd = chapter_pre_hook_cmd self.chapter_post_hook_cmd = chapter_post_hook_cmd - self.hook_infos: dict = {} self.cache_path = cache_path self.add_metadata = add_metadata + self.hook_infos: dict = {} # prepare everything self._prepare() diff --git a/mangadlp/cache.py b/mangadlp/cache.py index b6621d5..a2077b5 100644 --- a/mangadlp/cache.py +++ b/mangadlp/cache.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import Union +from typing import Dict, List, Union from loguru import logger as log @@ -33,7 +33,7 @@ class CacheDB: self.db_uuid_chapters: list = self.db_uuid_data.get("chapters") or [] - def _prepare_db(self): + def _prepare_db(self) -> None: if self.db_path.exists(): return # create empty cache @@ -44,11 +44,11 @@ class CacheDB: log.error("Can't create db-file") raise exc - def _read_db(self) -> dict: + def _read_db(self) -> Dict[str, dict]: log.info(f"Reading cache-db: {self.db_path}") try: db_txt = self.db_path.read_text(encoding="utf8") - db_dict: dict = json.loads(db_txt) + db_dict: dict[str, dict] = json.loads(db_txt) except Exception as exc: log.error("Can't load cache-db") raise exc @@ -73,7 +73,7 @@ class CacheDB: raise exc -def sort_chapters(chapters: list) -> list: +def sort_chapters(chapters: list) -> List[str]: try: sorted_list = sorted(chapters, key=float) except Exception: diff --git a/mangadlp/cli.py b/mangadlp/cli.py index 36803e9..b62e1cc 100644 --- a/mangadlp/cli.py +++ b/mangadlp/cli.py @@ -99,7 +99,7 @@ def readin_list(_ctx, _param, value) -> list: @click.option( "-p", "--path", - "path", + "download_path", type=click.Path(exists=False, writable=True, path_type=Path), default="downloads", required=False, @@ -109,7 +109,7 @@ def readin_list(_ctx, _param, value) -> list: @click.option( "-l", "--language", - "lang", + "language", type=str, default="en", required=False, @@ -127,7 +127,7 @@ def readin_list(_ctx, _param, value) -> list: ) @click.option( "--format", - "chapter_format", + "file_format", multiple=False, type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False), default="cbz", @@ -164,7 +164,7 @@ def readin_list(_ctx, _param, value) -> list: ) @click.option( "--wait", - "wait_time", + "download_wait", type=float, default=0.5, required=False, @@ -174,7 +174,7 @@ def readin_list(_ctx, _param, value) -> list: # hook options @click.option( "--hook-manga-pre", - "hook_manga_pre", + "manga_pre_hook_cmd", type=str, default=None, required=False, @@ -183,7 +183,7 @@ def readin_list(_ctx, _param, value) -> list: ) @click.option( "--hook-manga-post", - "hook_manga_post", + "manga_post_hook_cmd", type=str, default=None, required=False, @@ -192,7 +192,7 @@ def readin_list(_ctx, _param, value) -> list: ) @click.option( "--hook-chapter-pre", - "hook_chapter_pre", + "chapter_pre_hook_cmd", type=str, default=None, required=False, @@ -201,7 +201,7 @@ def readin_list(_ctx, _param, value) -> list: ) @click.option( "--hook-chapter-post", - "hook_chapter_post", + "chapter_post_hook_cmd", type=str, default=None, required=False, @@ -227,32 +227,16 @@ def readin_list(_ctx, _param, value) -> list: help="Enable/disable creation of metadata via ComicInfo.xml", ) @click.pass_context -def main( - ctx: click.Context, - url_uuid: str, - read_mangas: list, - verbosity: int, - chapters: str, - path: Path, - lang: str, - list_chapters: bool, - chapter_format: str, - name_format: str, - name_format_none: str, - forcevol: bool, - wait_time: float, - hook_manga_pre: str, - hook_manga_post: str, - hook_chapter_pre: str, - hook_chapter_post: str, - cache_path: str, - add_metadata: bool, -): # pylint: disable=too-many-locals +def main(ctx: click.Context, **kwargs) -> None: """ Script to download mangas from various sites """ + url_uuid: str = kwargs.pop("url_uuid") + read_mangas: list[str] = kwargs.pop("read_mangas") + verbosity: int = kwargs.pop("verbosity") + # set log level to INFO if not set if not verbosity: verbosity = 20 @@ -268,24 +252,7 @@ def main( for manga in requested_mangas: try: - mdlp = app.MangaDLP( - url_uuid=manga, - language=lang, - chapters=chapters, - list_chapters=list_chapters, - file_format=chapter_format, - name_format=name_format, - name_format_none=name_format_none, - forcevol=forcevol, - download_path=path, - download_wait=wait_time, - manga_pre_hook_cmd=hook_manga_pre, - manga_post_hook_cmd=hook_manga_post, - chapter_pre_hook_cmd=hook_chapter_pre, - chapter_post_hook_cmd=hook_chapter_post, - cache_path=cache_path, - add_metadata=add_metadata, - ) + mdlp = app.MangaDLP(url_uuid=manga, **kwargs) mdlp.get_manga() except (KeyboardInterrupt, Exception) as exc: # if only a single manga is requested and had an error, then exit diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index 3f7cc78..343a3cc 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any +from typing import Any, Dict, Tuple import xmltodict from loguru import logger as log @@ -8,7 +8,7 @@ METADATA_FILENAME = "ComicInfo.xml" METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml") # define metadata types, defaults and valid values. an empty list means no value check # {key: (type, default value, valid values)} -METADATA_TYPES: dict[str, tuple[type, Any, list]] = { +METADATA_TYPES: Dict[str, Tuple[type, Any, list]] = { "Title": (str, None, []), "Series": (str, None, []), "Number": (str, None, []), @@ -59,7 +59,7 @@ METADATA_TYPES: dict[str, tuple[type, Any, list]] = { } -def validate_metadata(metadata_in: dict) -> dict: +def validate_metadata(metadata_in: dict) -> Dict[str, dict]: log.info("Validating metadata") metadata_valid: dict[str, dict] = {"ComicInfo": {}} diff --git a/mangadlp/utils.py b/mangadlp/utils.py index 91fe552..4d9afc5 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -1,7 +1,7 @@ import re from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, List from zipfile import ZipFile from loguru import logger as log @@ -41,7 +41,7 @@ def make_pdf(chapter_path: Path) -> None: # create a list of chapters -def get_chapter_list(chapters: str, available_chapters: list) -> list: +def get_chapter_list(chapters: str, available_chapters: list) -> List[str]: # check if there are available chapter chapter_list: list[str] = [] for chapter in chapters.split(","): diff --git a/tests/test_21_full.py b/tests/test_21_full.py index 5809e4f..1cbb7f7 100644 --- a/tests/test_21_full.py +++ b/tests/test_21_full.py @@ -22,10 +22,12 @@ def wait_20s(): def test_full_api_mangadex(wait_20s): - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ) mdlp = app.MangaDLP( - url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie", + url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko", language="en", chapters="1", list_chapters=False, @@ -43,13 +45,15 @@ def test_full_api_mangadex(wait_20s): def test_full_with_input_cbz(wait_20s): - url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" language = "en" chapters = "1" file_format = "cbz" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ) command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2" script_path = "manga-dlp.py" os.system(f"python3 {script_path} {command_args}") @@ -61,13 +65,15 @@ def test_full_with_input_cbz(wait_20s): def test_full_with_input_cbz_info(wait_20s): - url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" language = "en" chapters = "1" file_format = "cbz" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ) command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2" script_path = "manga-dlp.py" os.system(f"python3 {script_path} {command_args}") @@ -82,13 +88,15 @@ def test_full_with_input_cbz_info(wait_20s): platform.machine() != "x86_64", reason="pdf only supported on amd64" ) def test_full_with_input_pdf(wait_20s): - url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" language = "en" chapters = "1" file_format = "pdf" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf" + ) command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2" script_path = "manga-dlp.py" os.system(f"python3 {script_path} {command_args}") @@ -100,14 +108,18 @@ def test_full_with_input_pdf(wait_20s): def test_full_with_input_folder(wait_20s): - url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" language = "en" chapters = "1" file_format = "" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1") - metadata_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1/ComicInfo.xml") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire" + ) + metadata_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml" + ) command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2" script_path = "manga-dlp.py" os.system(f"python3 {script_path} {command_args}") @@ -120,13 +132,15 @@ def test_full_with_input_folder(wait_20s): def test_full_with_input_skip_cbz(wait_10s): - url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" language = "en" chapters = "1" file_format = "cbz" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ) command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2" script_path = "manga-dlp.py" manga_path.mkdir(parents=True, exist_ok=True) @@ -140,13 +154,15 @@ def test_full_with_input_skip_cbz(wait_10s): def test_full_with_input_skip_folder(wait_10s): - url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" language = "en" chapters = "1" file_format = "" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire" + ) command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2" script_path = "manga-dlp.py" chapter_path.mkdir(parents=True, exist_ok=True) @@ -158,8 +174,12 @@ def test_full_with_input_skip_folder(wait_10s): assert chapter_path.is_dir() assert found_files == [] - assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists() - assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists() + assert not Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ).exists() + assert not Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip" + ).exists() # cleanup shutil.rmtree(manga_path, ignore_errors=True) @@ -170,12 +190,14 @@ def test_full_with_read_cbz(wait_20s): chapters = "1" file_format = "cbz" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ) command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2" script_path = "manga-dlp.py" url_list.write_text( - "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" ) os.system(f"python3 {script_path} {command_args}") @@ -192,14 +214,16 @@ def test_full_with_read_skip_cbz(wait_10s): chapters = "1" file_format = "cbz" download_path = "tests" - manga_path = Path("tests/Shikimori's Not Just a Cutie") - chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz") + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ) command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2" script_path = "manga-dlp.py" manga_path.mkdir(parents=True, exist_ok=True) chapter_path.touch() url_list.write_text( - "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" + "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko" ) os.system(f"python3 {script_path} {command_args}") @@ -209,6 +233,41 @@ def test_full_with_read_skip_cbz(wait_10s): shutil.rmtree(manga_path, ignore_errors=True) +def test_full_with_all_flags(wait_20s): + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = Path( + "tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz" + ) + cache_path = Path("tests/cache.json") + flags = [ + "-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko", + "--loglevel 10", + "-l en", + "-c 1", + "--path tests", + "--format cbz", + "--name-format 'Ch.{chapter_num} - {chapter_name}'", + "--name-format-none 0", + "--forcevol", + "--wait 2", + "--hook-manga-pre echo 0", + "--hook-manga-post 1", + "--hook-chapter-pre 2", + "--hook-chapter-post 3", + "--cache-path tests/cache.json", + "--add-metadata", + ] + script_path = "manga-dlp.py" + os.system(f"python3 {script_path} {' '.join(flags)}") + + assert manga_path.exists() and manga_path.is_dir() + assert chapter_path.exists() and chapter_path.is_file() + assert cache_path.exists() and cache_path.is_file() + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + cache_path.unlink(missing_ok=True) + + # def test_full_without_input(): # script_path = "manga-dlp.py" # assert os.system(f"python3 {script_path}") != 0 diff --git a/tests/test_22_all_flags.py b/tests/test_22_all_flags.py new file mode 100644 index 0000000..57f32fe --- /dev/null +++ b/tests/test_22_all_flags.py @@ -0,0 +1,54 @@ +import os +import platform +import shutil +import time +from pathlib import Path + +import pytest + +from mangadlp import app + + +@pytest.fixture +def wait_10s(): + print("sleeping 10 seconds because of api timeouts") + time.sleep(10) + + +@pytest.fixture +def wait_20s(): + print("sleeping 20 seconds because of api timeouts") + time.sleep(20) + + +def test_full_with_all_flags(wait_10s): + manga_path = Path("tests/Tomo-chan wa Onna no ko") + chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz" + cache_path = Path("tests/test_cache.json") + flags = [ + "-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko", + "--loglevel 10", + "-l en", + "-c 1", + "--path tests", + "--format cbz", + "--name-format 'Ch. {chapter_num} - {chapter_name}'", + "--name-format-none 0", + # "--forcevol", + "--wait 2", + "--hook-manga-pre 'echo 0'", + "--hook-manga-post 'echo 1'", + "--hook-chapter-pre 'echo 2'", + "--hook-chapter-post 'echo 3'", + "--cache-path tests/test_cache.json", + "--add-metadata", + ] + script_path = "manga-dlp.py" + os.system(f"python3 {script_path} {' '.join(flags)}") + + assert manga_path.exists() and manga_path.is_dir() + assert chapter_path.exists() and chapter_path.is_file() + assert cache_path.exists() and cache_path.is_file() + # cleanup + shutil.rmtree(manga_path, ignore_errors=True) + cache_path.unlink(missing_ok=True) diff --git a/tests/test_list2.txt b/tests/test_list2.txt index dda2d34..727c876 100644 --- a/tests/test_list2.txt +++ b/tests/test_list2.txt @@ -1 +1 @@ -https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie \ No newline at end of file +https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko \ No newline at end of file