update typo annotations and add new test
Some checks failed
ci/woodpecker/push/tests Pipeline failed
ci/woodpecker/pr/tests Pipeline failed
ci/woodpecker/pr/test_docker_arm64 unknown status
ci/woodpecker/pr/test_tox_arm64 unknown status
ci/woodpecker/pr/test_release unknown status
ci/woodpecker/pr/test_docker_amd64 unknown status
ci/woodpecker/pr/test_tox_amd64 unknown status

This commit is contained in:
Ivan Schaller 2023-02-15 20:17:22 +01:00
parent ef937f4ed0
commit ce6ebc4291
Signed by: olofvndrhr
GPG key ID: 2A6BE07D99C8C205
10 changed files with 185 additions and 100 deletions

View file

@ -18,6 +18,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- Added `xmltodict` as a package requirement
- Cache now also saves the manga title
- New tests
- More typo annotations for function, compatible with python3.8
### Fixed

View file

@ -1,5 +1,4 @@
# api template for manga-dlp
from typing import Any
class YourAPI:

View file

@ -17,18 +17,23 @@ class MangaDLP:
After initialization, start the script with the function get_manga().
Args:
url_uuid (str): URL or UUID of the manga
language (str): Manga language with country codes. "en" --> english
chapters (str): Chapters to download, "all" for every chapter available
list_chapters (bool): List all available chapters and exit
file_format (str): Archive format to create. An empty string means don't archive the folder
forcevol (bool): Force naming of volumes. Useful for mangas where chapters reset each volume
download_path (str/Path): Download path. Defaults to '<script_dir>/downloads'
download_wait (float): Time to wait for each picture to download in seconds
url_uuid: URL or UUID of the manga
language: Manga language with country codes. "en" --> english
chapters: Chapters to download, "all" for every chapter available
list_chapters: List all available chapters and exit
file_format: Archive format to create. An empty string means don't archive the folder
forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
download_path: Download path. Defaults to '<script_dir>/downloads'
download_wait: Time to wait for each picture to download in seconds
manga_pre_hook_cmd: Command(s) to before after each manga
manga_post_hook_cmd: Command(s) to run after each manga
chapter_pre_hook_cmd: Command(s) to run before each chapter
chapter_post_hook_cmd: Command(s) to run after each chapter
cache_path: Path to the json cache. If emitted, no cache is used
add_metadata: Flag to toggle creation & inclusion of metadata
"""
def __init__(
def __init__( # pylint: disable=too-many-locals
self,
url_uuid: str,
language: str = "en",
@ -62,9 +67,9 @@ class MangaDLP:
self.manga_post_hook_cmd = manga_post_hook_cmd
self.chapter_pre_hook_cmd = chapter_pre_hook_cmd
self.chapter_post_hook_cmd = chapter_post_hook_cmd
self.hook_infos: dict = {}
self.cache_path = cache_path
self.add_metadata = add_metadata
self.hook_infos: dict = {}
# prepare everything
self._prepare()

View file

@ -1,6 +1,6 @@
import json
from pathlib import Path
from typing import Union
from typing import Dict, List, Union
from loguru import logger as log
@ -33,7 +33,7 @@ class CacheDB:
self.db_uuid_chapters: list = self.db_uuid_data.get("chapters") or []
def _prepare_db(self):
def _prepare_db(self) -> None:
if self.db_path.exists():
return
# create empty cache
@ -44,11 +44,11 @@ class CacheDB:
log.error("Can't create db-file")
raise exc
def _read_db(self) -> dict:
def _read_db(self) -> Dict[str, dict]:
log.info(f"Reading cache-db: {self.db_path}")
try:
db_txt = self.db_path.read_text(encoding="utf8")
db_dict: dict = json.loads(db_txt)
db_dict: dict[str, dict] = json.loads(db_txt)
except Exception as exc:
log.error("Can't load cache-db")
raise exc
@ -73,7 +73,7 @@ class CacheDB:
raise exc
def sort_chapters(chapters: list) -> list:
def sort_chapters(chapters: list) -> List[str]:
try:
sorted_list = sorted(chapters, key=float)
except Exception:

View file

@ -99,7 +99,7 @@ def readin_list(_ctx, _param, value) -> list:
@click.option(
"-p",
"--path",
"path",
"download_path",
type=click.Path(exists=False, writable=True, path_type=Path),
default="downloads",
required=False,
@ -109,7 +109,7 @@ def readin_list(_ctx, _param, value) -> list:
@click.option(
"-l",
"--language",
"lang",
"language",
type=str,
default="en",
required=False,
@ -127,7 +127,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--format",
"chapter_format",
"file_format",
multiple=False,
type=click.Choice(["cbz", "cbr", "zip", "pdf", ""], case_sensitive=False),
default="cbz",
@ -164,7 +164,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--wait",
"wait_time",
"download_wait",
type=float,
default=0.5,
required=False,
@ -174,7 +174,7 @@ def readin_list(_ctx, _param, value) -> list:
# hook options
@click.option(
"--hook-manga-pre",
"hook_manga_pre",
"manga_pre_hook_cmd",
type=str,
default=None,
required=False,
@ -183,7 +183,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--hook-manga-post",
"hook_manga_post",
"manga_post_hook_cmd",
type=str,
default=None,
required=False,
@ -192,7 +192,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--hook-chapter-pre",
"hook_chapter_pre",
"chapter_pre_hook_cmd",
type=str,
default=None,
required=False,
@ -201,7 +201,7 @@ def readin_list(_ctx, _param, value) -> list:
)
@click.option(
"--hook-chapter-post",
"hook_chapter_post",
"chapter_post_hook_cmd",
type=str,
default=None,
required=False,
@ -227,32 +227,16 @@ def readin_list(_ctx, _param, value) -> list:
help="Enable/disable creation of metadata via ComicInfo.xml",
)
@click.pass_context
def main(
ctx: click.Context,
url_uuid: str,
read_mangas: list,
verbosity: int,
chapters: str,
path: Path,
lang: str,
list_chapters: bool,
chapter_format: str,
name_format: str,
name_format_none: str,
forcevol: bool,
wait_time: float,
hook_manga_pre: str,
hook_manga_post: str,
hook_chapter_pre: str,
hook_chapter_post: str,
cache_path: str,
add_metadata: bool,
): # pylint: disable=too-many-locals
def main(ctx: click.Context, **kwargs) -> None:
"""
Script to download mangas from various sites
"""
url_uuid: str = kwargs.pop("url_uuid")
read_mangas: list[str] = kwargs.pop("read_mangas")
verbosity: int = kwargs.pop("verbosity")
# set log level to INFO if not set
if not verbosity:
verbosity = 20
@ -268,24 +252,7 @@ def main(
for manga in requested_mangas:
try:
mdlp = app.MangaDLP(
url_uuid=manga,
language=lang,
chapters=chapters,
list_chapters=list_chapters,
file_format=chapter_format,
name_format=name_format,
name_format_none=name_format_none,
forcevol=forcevol,
download_path=path,
download_wait=wait_time,
manga_pre_hook_cmd=hook_manga_pre,
manga_post_hook_cmd=hook_manga_post,
chapter_pre_hook_cmd=hook_chapter_pre,
chapter_post_hook_cmd=hook_chapter_post,
cache_path=cache_path,
add_metadata=add_metadata,
)
mdlp = app.MangaDLP(url_uuid=manga, **kwargs)
mdlp.get_manga()
except (KeyboardInterrupt, Exception) as exc:
# if only a single manga is requested and had an error, then exit

View file

@ -1,5 +1,5 @@
from pathlib import Path
from typing import Any
from typing import Any, Dict, Tuple
import xmltodict
from loguru import logger as log
@ -8,7 +8,7 @@ METADATA_FILENAME = "ComicInfo.xml"
METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml")
# define metadata types, defaults and valid values. an empty list means no value check
# {key: (type, default value, valid values)}
METADATA_TYPES: dict[str, tuple[type, Any, list]] = {
METADATA_TYPES: Dict[str, Tuple[type, Any, list]] = {
"Title": (str, None, []),
"Series": (str, None, []),
"Number": (str, None, []),
@ -59,7 +59,7 @@ METADATA_TYPES: dict[str, tuple[type, Any, list]] = {
}
def validate_metadata(metadata_in: dict) -> dict:
def validate_metadata(metadata_in: dict) -> Dict[str, dict]:
log.info("Validating metadata")
metadata_valid: dict[str, dict] = {"ComicInfo": {}}

View file

@ -1,7 +1,7 @@
import re
from datetime import datetime
from pathlib import Path
from typing import Any
from typing import Any, List
from zipfile import ZipFile
from loguru import logger as log
@ -41,7 +41,7 @@ def make_pdf(chapter_path: Path) -> None:
# create a list of chapters
def get_chapter_list(chapters: str, available_chapters: list) -> list:
def get_chapter_list(chapters: str, available_chapters: list) -> List[str]:
# check if there are available chapter
chapter_list: list[str] = []
for chapter in chapters.split(","):

View file

@ -22,10 +22,12 @@ def wait_20s():
def test_full_api_mangadex(wait_20s):
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
mdlp = app.MangaDLP(
url_uuid="https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie",
url_uuid="https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
language="en",
chapters="1",
list_chapters=False,
@ -43,13 +45,15 @@ def test_full_api_mangadex(wait_20s):
def test_full_with_input_cbz(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -61,13 +65,15 @@ def test_full_with_input_cbz(wait_20s):
def test_full_with_input_cbz_info(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -82,13 +88,15 @@ def test_full_with_input_cbz_info(wait_20s):
platform.machine() != "x86_64", reason="pdf only supported on amd64"
)
def test_full_with_input_pdf(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "pdf"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.pdf")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.pdf"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -100,14 +108,18 @@ def test_full_with_input_pdf(wait_20s):
def test_full_with_input_folder(wait_20s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
metadata_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1/ComicInfo.xml")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire"
)
metadata_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire/ComicInfo.xml"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {command_args}")
@ -120,13 +132,15 @@ def test_full_with_input_folder(wait_20s):
def test_full_with_input_skip_cbz(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
@ -140,13 +154,15 @@ def test_full_with_input_skip_cbz(wait_10s):
def test_full_with_input_skip_folder(wait_10s):
url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
url_uuid = "https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
language = "en"
chapters = "1"
file_format = ""
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire"
)
command_args = f"-u {url_uuid} -l {language} -c {chapters} --path {download_path} --format '{file_format}' --debug --wait 2"
script_path = "manga-dlp.py"
chapter_path.mkdir(parents=True, exist_ok=True)
@ -158,8 +174,12 @@ def test_full_with_input_skip_folder(wait_10s):
assert chapter_path.is_dir()
assert found_files == []
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz").exists()
assert not Path("tests/Shikimori's Not Just a Cutie/Ch. 1.zip").exists()
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
).exists()
assert not Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.zip"
).exists()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
@ -170,12 +190,14 @@ def test_full_with_read_cbz(wait_20s):
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
url_list.write_text(
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
os.system(f"python3 {script_path} {command_args}")
@ -192,14 +214,16 @@ def test_full_with_read_skip_cbz(wait_10s):
chapters = "1"
file_format = "cbz"
download_path = "tests"
manga_path = Path("tests/Shikimori's Not Just a Cutie")
chapter_path = Path("tests/Shikimori's Not Just a Cutie/Ch. 1.cbz")
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
command_args = f"--read {str(url_list)} -l {language} -c {chapters} --path {download_path} --format {file_format} --debug --wait 2"
script_path = "manga-dlp.py"
manga_path.mkdir(parents=True, exist_ok=True)
chapter_path.touch()
url_list.write_text(
"https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie"
"https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko"
)
os.system(f"python3 {script_path} {command_args}")
@ -209,6 +233,41 @@ def test_full_with_read_skip_cbz(wait_10s):
shutil.rmtree(manga_path, ignore_errors=True)
def test_full_with_all_flags(wait_20s):
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = Path(
"tests/Tomo-chan wa Onna no ko/Ch. 1 - Once In A Life Time Misfire.cbz"
)
cache_path = Path("tests/cache.json")
flags = [
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
"--loglevel 10",
"-l en",
"-c 1",
"--path tests",
"--format cbz",
"--name-format 'Ch.{chapter_num} - {chapter_name}'",
"--name-format-none 0",
"--forcevol",
"--wait 2",
"--hook-manga-pre echo 0",
"--hook-manga-post 1",
"--hook-chapter-pre 2",
"--hook-chapter-post 3",
"--cache-path tests/cache.json",
"--add-metadata",
]
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {' '.join(flags)}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
assert cache_path.exists() and cache_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
cache_path.unlink(missing_ok=True)
# def test_full_without_input():
# script_path = "manga-dlp.py"
# assert os.system(f"python3 {script_path}") != 0

View file

@ -0,0 +1,54 @@
import os
import platform
import shutil
import time
from pathlib import Path
import pytest
from mangadlp import app
@pytest.fixture
def wait_10s():
print("sleeping 10 seconds because of api timeouts")
time.sleep(10)
@pytest.fixture
def wait_20s():
print("sleeping 20 seconds because of api timeouts")
time.sleep(20)
def test_full_with_all_flags(wait_10s):
manga_path = Path("tests/Tomo-chan wa Onna no ko")
chapter_path = manga_path / "Ch. 1 - Once In A Life Time Misfire.cbz"
cache_path = Path("tests/test_cache.json")
flags = [
"-u https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko",
"--loglevel 10",
"-l en",
"-c 1",
"--path tests",
"--format cbz",
"--name-format 'Ch. {chapter_num} - {chapter_name}'",
"--name-format-none 0",
# "--forcevol",
"--wait 2",
"--hook-manga-pre 'echo 0'",
"--hook-manga-post 'echo 1'",
"--hook-chapter-pre 'echo 2'",
"--hook-chapter-post 'echo 3'",
"--cache-path tests/test_cache.json",
"--add-metadata",
]
script_path = "manga-dlp.py"
os.system(f"python3 {script_path} {' '.join(flags)}")
assert manga_path.exists() and manga_path.is_dir()
assert chapter_path.exists() and chapter_path.is_file()
assert cache_path.exists() and cache_path.is_file()
# cleanup
shutil.rmtree(manga_path, ignore_errors=True)
cache_path.unlink(missing_ok=True)

View file

@ -1 +1 @@
https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie
https://mangadex.org/title/76ee7069-23b4-493c-bc44-34ccbf3051a8/tomo-chan-wa-onna-no-ko