From 2ad0c575a7c42e1ae8a9ad25ee5cc149d37e814d Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 12:23:50 +0100 Subject: [PATCH 01/19] switch to ruff and update justfile Signed-off-by: Ivan Schaller --- .woodpecker/test_release.yml | 2 +- .woodpecker/test_tox_amd64.yml | 2 +- .woodpecker/test_tox_arm64.yml | 2 +- .woodpecker/tests.yml | 39 ++-------- contrib/requirements_dev.txt | 5 +- justfile | 53 +++++--------- pyproject.toml | 130 ++++++++++++++++++++------------- tox.ini | 5 -- 8 files changed, 111 insertions(+), 127 deletions(-) diff --git a/.woodpecker/test_release.yml b/.woodpecker/test_release.yml index ef64caa..50409cc 100644 --- a/.woodpecker/test_release.yml +++ b/.woodpecker/test_release.yml @@ -26,7 +26,7 @@ pipeline: branch: master event: pull_request commands: - - python3 -m hatch build --clean + - just test_build # create release-notes test-create-release-notes: diff --git a/.woodpecker/test_tox_amd64.yml b/.woodpecker/test_tox_amd64.yml index 342fe15..f365a7c 100644 --- a/.woodpecker/test_tox_amd64.yml +++ b/.woodpecker/test_tox_amd64.yml @@ -26,4 +26,4 @@ pipeline: branch: master event: pull_request commands: - - python3 -m tox + - just test_tox diff --git a/.woodpecker/test_tox_arm64.yml b/.woodpecker/test_tox_arm64.yml index 87a9a20..a203c1d 100644 --- a/.woodpecker/test_tox_arm64.yml +++ b/.woodpecker/test_tox_arm64.yml @@ -29,4 +29,4 @@ pipeline: - grep -v img2pdf contrib/requirements_dev.txt > contrib/requirements_dev_arm64.txt - rm -f contrib/requirements_dev.txt - mv contrib/requirements_dev_arm64.txt contrib/requirements_dev.txt - - python3 -m tox + - just test_tox diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml index 72c95eb..ead19b0 100644 --- a/.woodpecker/tests.yml +++ b/.woodpecker/tests.yml @@ -17,51 +17,28 @@ pipeline: image: cr.44net.ch/ci-plugins/tests pull: true commands: - - shfmt -d -i 4 -bn -ci -sr . + - just test_shfmt # check code style - python test-black: image: cr.44net.ch/ci-plugins/tests pull: true commands: - - python3 -m black --check --diff . - - # check imports - python - test-isort: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - python3 -m isort --check-only --diff . - - # check unused and missing imports - python - test-autoflake: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/ - - python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/ + - just test_black # check static typing - python test-mypy: image: cr.44net.ch/ci-plugins/tests pull: true commands: - - python3 -m mypy --install-types --non-interactive mangadlp/ + - just test_mypy - # mccabe, pycodestyle, pyflakes tests - python - test-pylama: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - python3 -m pylama mangadlp/ - - # pylint test - python + # ruff test - python test-pylint: image: cr.44net.ch/ci-plugins/tests pull: true commands: - - python3 -m pip install -r requirements.txt - - python3 -m pylint --fail-under 9 mangadlp/ + - just test_ruff # test mkdocs generation test-mkdocs: @@ -72,14 +49,14 @@ pipeline: - cd docs || exit 1 - python3 -m mkdocs build --strict - # test code with different python versions - python + # test code with pytest - python test-tox-pytest: when: event: [ push ] image: cr.44net.ch/ci-plugins/tests pull: true commands: - - python3 -m tox -e basic + - just test_pytest # generate coverage report - python test-tox-coverage: @@ -89,7 +66,7 @@ pipeline: image: cr.44net.ch/ci-plugins/tests pull: true commands: - - python3 -m tox -e coverage + - just test_coverage # analyse code with sonarqube and upload it sonarqube-analysis: diff --git a/contrib/requirements_dev.txt b/contrib/requirements_dev.txt index a2aa1f6..b8d5219 100644 --- a/contrib/requirements_dev.txt +++ b/contrib/requirements_dev.txt @@ -14,9 +14,6 @@ hatchling>=1.11.0 pytest>=7.0.0 coverage>=6.3.1 black>=22.1.0 -isort>=5.10.0 -pylint>=2.13.0 mypy>=0.940 tox>=3.24.5 -autoflake>=1.4 -pylama>=8.3.8 +ruff>=0.0.247 diff --git a/justfile b/justfile index cd35d4d..1fbbd5f 100755 --- a/justfile +++ b/justfile @@ -68,45 +68,39 @@ create_venv: @python3 -m venv venv install_deps: + @echo "installing dependencies" + @pip3 install -r requirements.txt + +install_deps_dev: @echo "installing dependencies" @pip3 install -r contrib/requirements_dev.txt test_shfmt: - @find . -type f \( -name "**.sh" -and -not -path "./venv/*" -and -not -path "./.tox/*" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+; + @find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+; test_black: @python3 -m black --check --diff . -test_isort: - @python3 -m isort --check-only --diff . - test_mypy: - @python3 -m mypy --install-types --non-interactive mangadlp/ + @python3 -m mypy --install-types --non-interactive --ignore-missing-imports . + +test_ruff: + @python3 -m ruff --diff . + +test_ci_conf: + @woodpecker-cli lint .woodpecker/ test_pytest: @python3 -m tox -e basic -test_autoflake: - @python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/ - @python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/ - -test_pylama: - @python3 -m pylama --options tox.ini mangadlp/ - -test_pylint: - @python3 -m pylint --fail-under 9 mangadlp/ +test_coverage: + @python3 -m tox -e coverage test_tox: @python3 -m tox -test_tox_coverage: - @python3 -m tox -e coverage - test_build: - @python3 -m hatch build - -test_ci_conf: - @woodpecker-cli lint .woodpecker/ + @python3 -m hatch build --clean test_docker_build: @docker build . -f docker/Dockerfile.amd64 -t manga-dlp:test @@ -123,11 +117,8 @@ lint: -just test_ci_conf just test_shfmt just test_black - just test_isort just test_mypy - just test_autoflake - just test_pylama - just test_pylint + just test_ruff @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" tests: @@ -135,11 +126,8 @@ tests: -just test_ci_conf just test_shfmt just test_black - just test_isort just test_mypy - just test_autoflake - just test_pylama - just test_pylint + just test_ruff just test_pytest @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" @@ -148,13 +136,10 @@ tests_full: -just test_ci_conf just test_shfmt just test_black - just test_isort just test_mypy - just test_autoflake - just test_pylama - just test_pylint + just test_ruff just test_build just test_tox - just test_tox_coverage + just test_coverage just test_docker_build @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" diff --git a/pyproject.toml b/pyproject.toml index 9762d3c..49f557a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling>=1.11.0"] +requires = ["hatchling>=1.11.0"] build-backend = "hatchling.build" [project] @@ -9,14 +9,8 @@ description = "A cli manga downloader" readme = "README.md" license = "MIT" requires-python = ">=3.8" -authors = [ - { name = "Ivan Schaller", email = "ivan@schaller.sh" }, -] -keywords = [ - "manga", - "downloader", - "mangadex", -] +authors = [{ name = "Ivan Schaller", email = "ivan@schaller.sh" }] +keywords = ["manga", "downloader", "mangadex"] classifiers = [ "License :: OSI Approved :: MIT License", "Natural Language :: English", @@ -30,17 +24,17 @@ dependencies = [ "loguru>=0.6.0", "click>=8.1.3", "click-option-group>=0.5.5", - "xmltodict>=0.13.0" + "xmltodict>=0.13.0", ] [project.urls] Homepage = "https://github.com/olofvndrhr/manga-dlp" -History = "https://github.com/olofvndrhr/manga-dlp/commits/master" -Tracker = "https://github.com/olofvndrhr/manga-dlp/issues" -Source = "https://github.com/olofvndrhr/manga-dlp" +History = "https://github.com/olofvndrhr/manga-dlp/commits/master" +Tracker = "https://github.com/olofvndrhr/manga-dlp/issues" +Source = "https://github.com/olofvndrhr/manga-dlp" [project.scripts] -mangadlp = "mangadlp.cli:main" +mangadlp = "mangadlp.cli:main" manga-dlp = "mangadlp.cli:main" [tool.hatch.version] @@ -69,44 +63,89 @@ dependencies = [ "pytest>=7.0.0", "coverage>=6.3.1", "black>=22.1.0", - "isort>=5.10.0", - "pylint>=2.13.0", "mypy>=0.940", "tox>=3.24.5", - "autoflake>=1.4", - "pylama>=8.3.8", + "ruff>=0.0.247", ] -[tool.isort] -py_version = 39 -skip_gitignore = true -line_length = 88 -profile = "black" -multi_line_output = 3 -include_trailing_comma = true -use_parentheses = true +# mypy [tool.mypy] -python_version = "3.9" -disallow_untyped_defs = false -follow_imports = "normal" +#strict = true +python_version = "3.9" +disallow_untyped_defs = false +follow_imports = "normal" ignore_missing_imports = true -warn_no_return = false -warn_unused_ignores = true -show_error_context = true -show_column_numbers = true -show_error_codes = true -pretty = true -no_implicit_optional = false +warn_no_return = false +warn_unused_ignores = true +show_error_context = true +show_column_numbers = true +show_error_codes = true +pretty = true +no_implicit_optional = false -[tool.pytest.ini_options] -pythonpath = [ - "." +# ruff + +[tool.ruff] +target-version = "py39" +select = [ + "E", # pycodetyle err + "W", # pycodetyle warn + "D", # pydocstyle + "C90", # mccabe + "I", # isort + "PLE", # pylint err + "PLW", # pylint warn + "PLC", # pylint convention + "PLR", # pylint refactor + "F", # pyflakes + "RUF", # ruff specific +] +line-length = 88 +fix = true +format = "grouped" +ignore-init-module-imports = true +respect-gitignore = true +ignore = ["E501", "D103", "D100"] +exclude = [ + ".direnv", + ".git", + ".mypy_cache", + ".ruff_cache", + ".svn", + ".venv", + "venv", + "__pypackages__", + "build", + "dist", + "venv", ] +[tool.ruff.per-file-ignores] +"__init__.py" = ["D104"] + +[tool.ruff.pylint] +max-args = 10 + +[tool.ruff.mccabe] +max-complexity = 10 + +[tool.ruff.pydocstyle] +convention = "google" + +[tool.ruff.pycodestyle] +max-doc-length = 88 + +# pytest + +[tool.pytest.ini_options] +pythonpath = ["."] + +# coverage + [tool.coverage.run] -source = ["mangadlp"] -branch = true +source = ["mangadlp"] +branch = true command_line = "-m pytest --exitfirst" [tool.coverage.report] @@ -127,12 +166,3 @@ exclude_lines = [ "@(abc.)?abstractmethod", ] ignore_errors = true - -[tool.pylint.main] -py-version = "3.9" - -[tool.pylint.logging] -logging-modules = ["logging", "loguru"] -disable = "C0301, C0114, C0116, W0703, R0902, R0913, E0401, W1203" -good-names = "r" -logging-format-style = "new" diff --git a/tox.ini b/tox.ini index 22d02c8..a32eeb8 100644 --- a/tox.ini +++ b/tox.ini @@ -24,8 +24,3 @@ commands = coverage erase coverage run coverage xml -i - -[pylama] -format = pycodestyle -linters = mccabe,pycodestyle,pyflakes -ignore = E501,C901,C0301 From 5e28cb10881db45d0632ca769ab33d01bc396a04 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 12:29:25 +0100 Subject: [PATCH 02/19] format with ruff Signed-off-by: Ivan Schaller --- contrib/api_template.py | 20 ++++++++------------ mangadlp/api/mangadex.py | 3 ++- mangadlp/app.py | 4 ++-- mangadlp/cli.py | 6 +----- mangadlp/hooks.py | 6 ++++-- mangadlp/logger.py | 4 +--- mangadlp/metadata.py | 2 +- tests/test_04_input.py | 3 --- tests/test_05_hooks.py | 10 +++++----- tests/test_06_cache.py | 2 +- tests/test_07_metadata.py | 2 +- tests/test_11_api_mangadex.py | 3 +-- 12 files changed, 27 insertions(+), 38 deletions(-) diff --git a/contrib/api_template.py b/contrib/api_template.py index aa95174..89ed331 100644 --- a/contrib/api_template.py +++ b/contrib/api_template.py @@ -3,7 +3,8 @@ class YourAPI: """Your API Class. - Get infos for a manga from example.org + + Get infos for a manga from example.org. Args: url_uuid (str): URL or UUID of the manga @@ -23,9 +24,7 @@ class YourAPI: img_base_url = "https://uploads.mangadex.org" def __init__(self, url_uuid, language, forcevol): - """ - get infos to initiate class - """ + """get infos to initiate class.""" self.api_name = "Your API Name" self.url_uuid = url_uuid @@ -67,8 +66,7 @@ class YourAPI: } def get_chapter_images(chapter: str, download_wait: float) -> list: - """ - Get chapter images as a list (full links) + """Get chapter images as a list (full links). Args: chapter: The chapter number (chapter data index) @@ -77,7 +75,6 @@ class YourAPI: Returns: The list of urls of the page images """ - # example return [ "https://abc.def/image/123.png", @@ -85,10 +82,10 @@ class YourAPI: "https://abc.def/image/12345.png", ] - def create_metadata(self, chapter: str) -> dict: - """ - Get metadata with correct keys for ComicInfo.xml - Provide as much metadata as possible. empty/false values will be ignored + def create_metadata(chapter: str) -> dict: + """Get metadata with correct keys for ComicInfo.xml. + + Provide as much metadata as possible. empty/false values will be ignored. Args: chapter: The chapter number (chapter data index) @@ -96,7 +93,6 @@ class YourAPI: Returns: The metadata as a dict """ - # metadata types. have to be valid # {key: (type, default value, valid values)} { diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index 12abf93..a6e3f11 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -9,7 +9,8 @@ from mangadlp import utils class Mangadex: """Mangadex API Class. - Get infos for a manga from mangadex.org + + Get infos for a manga from mangadex.org. Args: url_uuid (str): URL or UUID of the manga diff --git a/mangadlp/app.py b/mangadlp/app.py index 1fb58a8..439deb3 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -14,7 +14,7 @@ from mangadlp.utils import get_file_format def match_api(url_uuid: str) -> type: - """Match the correct api class from a string + """Match the correct api class from a string. Args: url_uuid: url/uuid to check @@ -22,7 +22,6 @@ def match_api(url_uuid: str) -> type: Returns: The class of the API to use """ - # apis to check apis: list[tuple[str, re.Pattern, type]] = [ ( @@ -53,6 +52,7 @@ def match_api(url_uuid: str) -> type: class MangaDLP: """Download Mangas from supported sites. + After initialization, start the script with the function get_manga(). Args: diff --git a/mangadlp/cli.py b/mangadlp/cli.py index b62e1cc..0da36a2 100644 --- a/mangadlp/cli.py +++ b/mangadlp/cli.py @@ -228,11 +228,7 @@ def readin_list(_ctx, _param, value) -> list: ) @click.pass_context def main(ctx: click.Context, **kwargs) -> None: - """ - Script to download mangas from various sites - - """ - + """Script to download mangas from various sites.""" url_uuid: str = kwargs.pop("url_uuid") read_mangas: list[str] = kwargs.pop("read_mangas") verbosity: int = kwargs.pop("verbosity") diff --git a/mangadlp/hooks.py b/mangadlp/hooks.py index 31c702a..514af64 100644 --- a/mangadlp/hooks.py +++ b/mangadlp/hooks.py @@ -5,7 +5,10 @@ from loguru import logger as log def run_hook(command: str, hook_type: str, **kwargs) -> int: - """ + """Run a command. + + Run a command with subprocess.run and add kwargs to the environment. + Args: command (str): command to run hook_type (str): type of the hook @@ -14,7 +17,6 @@ def run_hook(command: str, hook_type: str, **kwargs) -> int: Returns: exit_code (int): exit code of command """ - # check if hook commands are empty if not command or command == "None": log.debug(f"Hook '{hook_type}' empty. Not running") diff --git a/mangadlp/logger.py b/mangadlp/logger.py index 576baa5..f332ed4 100644 --- a/mangadlp/logger.py +++ b/mangadlp/logger.py @@ -8,9 +8,7 @@ LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | [{level: <7}] [{name: # from loguru docs class InterceptHandler(logging.Handler): - """ - Intercept python logging messages and log them via loguru.logger - """ + """Intercept python logging messages and log them via loguru.logger.""" def emit(self, record): # Get corresponding Loguru level if it exists diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index 343a3cc..3e31463 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -84,7 +84,7 @@ def validate_metadata(metadata_in: dict) -> Dict[str, dict]: # check if metadata type is correct log.debug(f"Key:{key} -> value={type(md_to_check)} -> check={metadata_type}") - if not isinstance(md_to_check, metadata_type): # noqa + if not isinstance(md_to_check, metadata_type): log.warning( f"Metadata has wrong type: {key}:{metadata_type} -> {md_to_check}" ) diff --git a/tests/test_04_input.py b/tests/test_04_input.py index 093558d..a4e4636 100644 --- a/tests/test_04_input.py +++ b/tests/test_04_input.py @@ -16,8 +16,6 @@ def test_read_and_url(): def test_no_read_and_url(): - url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" - link_file = "tests/testfile.txt" language = "en" chapters = "1" file_format = "cbz" @@ -30,7 +28,6 @@ def test_no_read_and_url(): def test_no_chaps(): url_uuid = "https://mangadex.org/title/0aea9f43-d4a9-4bf7-bebc-550a512f9b95/shikimori-s-not-just-a-cutie" language = "en" - chapters = "" file_format = "cbz" download_path = "tests" command_args = f"-u {url_uuid} -l {language} --path {download_path} --format {file_format} --debug" diff --git a/tests/test_05_hooks.py b/tests/test_05_hooks.py index 8eb75ac..31ee074 100644 --- a/tests/test_05_hooks.py +++ b/tests/test_05_hooks.py @@ -40,7 +40,7 @@ def test_manga_pre_hook(wait_10s): manga_pre_hook, ] script_path = "manga-dlp.py" - command = ["python3", script_path] + command_args + command = ["python3", script_path, *command_args] assert subprocess.call(command) == 0 assert hook_file.is_file() @@ -72,7 +72,7 @@ def test_manga_post_hook(wait_10s): manga_post_hook, ] script_path = "manga-dlp.py" - command = ["python3", script_path] + command_args + command = ["python3", script_path, *command_args] assert subprocess.call(command) == 0 assert hook_file.is_file() @@ -104,7 +104,7 @@ def test_chapter_pre_hook(wait_10s): chapter_pre_hook, ] script_path = "manga-dlp.py" - command = ["python3", script_path] + command_args + command = ["python3", script_path, *command_args] assert subprocess.call(command) == 0 assert hook_file.is_file() @@ -136,7 +136,7 @@ def test_chapter_post_hook(wait_10s): chapter_post_hook, ] script_path = "manga-dlp.py" - command = ["python3", script_path] + command_args + command = ["python3", script_path, *command_args] assert subprocess.call(command) == 0 assert hook_file.is_file() @@ -176,7 +176,7 @@ def test_all_hooks(wait_10s): chapter_post_hook, ] script_path = "manga-dlp.py" - command = ["python3", script_path] + command_args + command = ["python3", script_path, *command_args] assert subprocess.call(command) == 0 assert Path("tests/manga-pre2.txt").is_file() diff --git a/tests/test_06_cache.py b/tests/test_06_cache.py index a3e065b..c46c96b 100644 --- a/tests/test_06_cache.py +++ b/tests/test_06_cache.py @@ -6,7 +6,7 @@ from mangadlp.cache import CacheDB def test_cache_creation(): cache_file = Path("cache.json") - cache = CacheDB(cache_file, "abc", "en", "test") + CacheDB(cache_file, "abc", "en", "test") assert cache_file.exists() cache_file.unlink() diff --git a/tests/test_07_metadata.py b/tests/test_07_metadata.py index 33ff523..c67b1fe 100644 --- a/tests/test_07_metadata.py +++ b/tests/test_07_metadata.py @@ -133,7 +133,7 @@ def test_metadata_chapter_validity(wait_20s): schema = xmlschema.XMLSchema("mangadlp/metadata/ComicInfo_v2.0.xsd") script_path = "manga-dlp.py" - command = ["python3", script_path] + command_args + command = ["python3", script_path, *command_args] assert subprocess.call(command) == 0 assert metadata_path.is_file() diff --git a/tests/test_11_api_mangadex.py b/tests/test_11_api_mangadex.py index 38ace21..fc28787 100644 --- a/tests/test_11_api_mangadex.py +++ b/tests/test_11_api_mangadex.py @@ -56,7 +56,7 @@ def test_alt_title_fallback(): forcevol = False test = Mangadex(url_uuid, language, forcevol) - assert test.manga_title == "Iruma à l’école des démons" + assert test.manga_title == "Iruma à l`école des démons" def test_chapter_infos(): @@ -206,7 +206,6 @@ def test_get_chapter_images(): test = Mangadex(url_uuid, language, forcevol) img_base_url = "https://uploads.mangadex.org" chapter_hash = "0752bc5db298beff6b932b9151dd8437" - chapter_uuid = "e86ec2c4-c5e4-4710-bfaa-7604f00939c7" chapter_num = "1" test_list = [ f"{img_base_url}/data/{chapter_hash}/x1-0deb4c9bfedd5be49e0a90cfb17cf343888239898c9e7451d569c0b3ea2971f4.jpg", From b5c5b97b16abaad942892068a13aa195b61bc032 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 12:32:30 +0100 Subject: [PATCH 03/19] fix ci task name Signed-off-by: Ivan Schaller --- .woodpecker/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml index ead19b0..bb8ddc6 100644 --- a/.woodpecker/tests.yml +++ b/.woodpecker/tests.yml @@ -34,7 +34,7 @@ pipeline: - just test_mypy # ruff test - python - test-pylint: + test-ruff: image: cr.44net.ch/ci-plugins/tests pull: true commands: From a8f4b25802424ab06013810ab1f6cd1673be0cc9 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 12:44:42 +0100 Subject: [PATCH 04/19] fix special character in test Signed-off-by: Ivan Schaller --- tests/test_11_api_mangadex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_11_api_mangadex.py b/tests/test_11_api_mangadex.py index fc28787..41b34cf 100644 --- a/tests/test_11_api_mangadex.py +++ b/tests/test_11_api_mangadex.py @@ -56,7 +56,7 @@ def test_alt_title_fallback(): forcevol = False test = Mangadex(url_uuid, language, forcevol) - assert test.manga_title == "Iruma à l`école des démons" + assert test.manga_title == "Iruma à l’école des démons" # noqa def test_chapter_infos(): From ef7a91486955918b0e79848681c2790814c29f38 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 12:52:34 +0100 Subject: [PATCH 05/19] update readme for ruff [CI SKIP] Signed-off-by: Ivan Schaller --- README.md | 20 ++++++++++---------- docs/pages/index.md | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f569f1d..bab78b9 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,13 @@ Code Analysis Meta [![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black) -[![Linter](https://img.shields.io/badge/linter-pylint-yellowgreen)](https://pylint.pycqa.org/en/latest/) +[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff) [![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy) -[![Imports](https://img.shields.io/badge/imports-isort-ef8336.svg)](https://github.com/pycqa/isort) [![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/) [![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy) [![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/) [![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/) + --- ## Description @@ -42,7 +42,7 @@ you just want the folder with all the pictures use the flag `--nocbz`. ## _Currently_ Supported sites -- [Mangadex.org](https://mangadex.org/) +- [Mangadex.org](https://mangadex.org/) ## Usage @@ -134,10 +134,10 @@ If you encounter any bugs, also just open an issue with a description of the pro ## TODO's -- Make docker container for easy distribution - --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp) -- Automate release - --> Done with woodpecker-ci -- Make pypi package - --> Done with release [2.1.7](https://pypi.org/project/manga-dlp/) -- Add more supported sites +- Make docker container for easy distribution + --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp) +- Automate release + --> Done with woodpecker-ci +- Make pypi package + --> Done with release [2.1.7](https://pypi.org/project/manga-dlp/) +- Add more supported sites diff --git a/docs/pages/index.md b/docs/pages/index.md index efca1e0..a808397 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -17,13 +17,13 @@ Code Analysis Meta [![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black) -[![Linter](https://img.shields.io/badge/linter-pylint-yellowgreen)](https://pylint.pycqa.org/en/latest/) +[![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff) [![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy) -[![Imports](https://img.shields.io/badge/imports-isort-ef8336.svg)](https://github.com/pycqa/isort) [![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/) [![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy) [![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/) [![Compatibility](https://img.shields.io/pypi/pyversions/manga-dlp)](https://pypi.org/project/manga-dlp/) + --- ## Description @@ -40,7 +40,7 @@ you just want the folder with all the pictures use the flag `--nocbz`. ## _Currently_ Supported sites -- [Mangadex.org](https://mangadex.org/) +- [Mangadex.org](https://mangadex.org/) ## Usage @@ -134,10 +134,10 @@ If you encounter any bugs, also just open an issue with a description of the pro ## TODO's -- Make docker container for easy distribution - --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp) -- Automate release - --> Done with woodpecker-ci -- Make pypi package - --> Done with release [2.1.7](https://pypi.org/project/manga-dlp/) -- Add more supported sites +- Make docker container for easy distribution + --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp) +- Automate release + --> Done with woodpecker-ci +- Make pypi package + --> Done with release [2.1.7](https://pypi.org/project/manga-dlp/) +- Add more supported sites From 03461b80bf60e9bc740f5fccd9a2f3329a23fcf2 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 16:21:03 +0100 Subject: [PATCH 06/19] switch to strict typing with pyright Signed-off-by: Ivan Schaller --- .woodpecker/tests.yml | 7 ++++++ contrib/requirements_dev.txt | 1 + justfile | 8 ++++++- mangadlp/api/mangadex.py | 46 ++++++++++++++++++++---------------- mangadlp/app.py | 20 ++++++++-------- mangadlp/cache.py | 12 ++++++---- mangadlp/cli.py | 21 ++++++++-------- mangadlp/downloader.py | 8 +++---- mangadlp/hooks.py | 3 ++- mangadlp/logger.py | 9 +++---- mangadlp/metadata.py | 12 ++++++---- mangadlp/utils.py | 6 ++--- pyproject.toml | 17 +++++++++++-- 13 files changed, 104 insertions(+), 66 deletions(-) diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml index bb8ddc6..06097c7 100644 --- a/.woodpecker/tests.yml +++ b/.woodpecker/tests.yml @@ -33,6 +33,13 @@ pipeline: commands: - just test_mypy + # check static typing - python + test-pyright: + image: cr.44net.ch/ci-plugins/tests + pull: true + commands: + - just test_pyright + # ruff test - python test-ruff: image: cr.44net.ch/ci-plugins/tests diff --git a/contrib/requirements_dev.txt b/contrib/requirements_dev.txt index b8d5219..680db66 100644 --- a/contrib/requirements_dev.txt +++ b/contrib/requirements_dev.txt @@ -17,3 +17,4 @@ black>=22.1.0 mypy>=0.940 tox>=3.24.5 ruff>=0.0.247 +pyright>=1.1.294 diff --git a/justfile b/justfile index 1fbbd5f..d2e0b38 100755 --- a/justfile +++ b/justfile @@ -82,7 +82,10 @@ test_black: @python3 -m black --check --diff . test_mypy: - @python3 -m mypy --install-types --non-interactive --ignore-missing-imports . + @python3 -m mypy --install-types --non-interactive --ignore-missing-imports mangadlp/ + +test_pyright: + @python3 -m pyright mangadlp/ test_ruff: @python3 -m ruff --diff . @@ -118,6 +121,7 @@ lint: just test_shfmt just test_black just test_mypy + just test_pyright just test_ruff @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" @@ -127,6 +131,7 @@ tests: just test_shfmt just test_black just test_mypy + just test_pyright just test_ruff just test_pytest @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" @@ -137,6 +142,7 @@ tests_full: just test_shfmt just test_black just test_mypy + just test_pyright just test_ruff just test_build just test_tox diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index a6e3f11..5b79460 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -1,5 +1,6 @@ import re from time import sleep +from typing import Any, Dict, List, Union import requests from loguru import logger as log @@ -65,10 +66,10 @@ class Mangadex: log.error("No valid UUID found") raise exc - return uuid + return uuid # pyright:ignore # make initial request - def get_manga_data(self) -> dict: + def get_manga_data(self) -> Dict[str, Any]: log.debug(f"Getting manga data for: {self.manga_uuid}") counter = 1 while counter <= 3: @@ -85,12 +86,14 @@ class Mangadex: counter += 1 else: break + + response_body: Dict[str, Dict[str, Any]] = response.json() # pyright:ignore # check if manga exists - if response.json()["result"] != "ok": + if response_body["result"] != "ok": # type:ignore log.error("Manga not found") raise KeyError - return response.json()["data"] + return response_body["data"] # get the title of the manga (and fix the filename) def get_manga_title(self) -> str: @@ -112,7 +115,7 @@ class Mangadex: if item.get(self.language): alt_title = item break - title = alt_title[self.language] + title = alt_title[self.language] # pyright:ignore except (KeyError, UnboundLocalError): log.warning( "Manga title also not found in alt titles. Falling back to english title" @@ -133,7 +136,7 @@ class Mangadex: timeout=10, ) try: - total_chapters = r.json()["total"] + total_chapters: int = r.json()["total"] except Exception as exc: log.error( "Error retrieving the chapters list. Did you specify a valid language code?" @@ -147,7 +150,7 @@ class Mangadex: return total_chapters # get chapter data like name, uuid etc - def get_chapter_data(self) -> dict: + def get_chapter_data(self) -> Dict[str, Dict[str, Union[str, int]]]: log.debug(f"Getting chapter data for: {self.manga_uuid}") api_sorting = "order[chapter]=asc&order[volume]=asc" # check for chapters in specified lang @@ -161,8 +164,9 @@ class Mangadex: f"{self.api_base_url}/manga/{self.manga_uuid}/feed?{api_sorting}&limit=500&offset={offset}&{self.api_additions}", timeout=10, ) - for chapter in r.json()["data"]: - attributes: dict = chapter["attributes"] + response_body: Dict[str, Any] = r.json() + for chapter in response_body["data"]: + attributes: Dict[str, Any] = chapter["attributes"] # chapter infos from feed chapter_num: str = attributes.get("chapter") or "" chapter_vol: str = attributes.get("volume") or "" @@ -201,10 +205,10 @@ class Mangadex: # increase offset for mangas with more than 500 chapters offset += 500 - return chapter_data + return chapter_data # type:ignore # get images for the chapter (mangadex@home) - def get_chapter_images(self, chapter: str, wait_time: float) -> list: + def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]: log.debug(f"Getting chapter images for: {self.manga_uuid}") athome_url = f"{self.api_base_url}/at-home/server" chapter_uuid = self.manga_chapter_data[chapter]["uuid"] @@ -238,11 +242,11 @@ class Mangadex: if api_error: return [] - chapter_hash = api_data["chapter"]["hash"] - chapter_img_data = api_data["chapter"]["data"] + chapter_hash = api_data["chapter"]["hash"] # pyright:ignore + chapter_img_data = api_data["chapter"]["data"] # pyright:ignore # get list of image urls - image_urls = [] + image_urls: List[str] = [] for image in chapter_img_data: image_urls.append(f"{self.img_base_url}/data/{chapter_hash}/{image}") @@ -251,12 +255,12 @@ class Mangadex: return image_urls # create list of chapters - def create_chapter_list(self) -> list: + def create_chapter_list(self) -> List[str]: log.debug(f"Creating chapter list for: {self.manga_uuid}") - chapter_list = [] + chapter_list: List[str] = [] for data in self.manga_chapter_data.values(): - chapter_number: str = data["chapter"] - volume_number: str = data["volume"] + chapter_number: str = data["chapter"] # type:ignore + volume_number: str = data["volume"] # type:ignore if self.forcevol: chapter_list.append(f"{volume_number}:{chapter_number}") else: @@ -264,12 +268,12 @@ class Mangadex: return chapter_list - def create_metadata(self, chapter: str) -> dict: + def create_metadata(self, chapter: str) -> Dict[str, Union[str, int, None]]: log.info("Creating metadata from api") chapter_data = self.manga_chapter_data[chapter] try: - volume = int(chapter_data.get("volume")) + volume = int(chapter_data["volume"]) except (ValueError, TypeError): volume = None metadata = { @@ -285,4 +289,4 @@ class Mangadex: "Web": f"https://mangadex.org/title/{self.manga_uuid}", } - return metadata + return metadata # pyright:ignore diff --git a/mangadlp/app.py b/mangadlp/app.py index 439deb3..4b8ad91 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -1,7 +1,7 @@ import re import shutil from pathlib import Path -from typing import Any, Union +from typing import Any, Dict, List, Tuple, Union from loguru import logger as log @@ -23,7 +23,7 @@ def match_api(url_uuid: str) -> type: The class of the API to use """ # apis to check - apis: list[tuple[str, re.Pattern, type]] = [ + apis: List[Tuple[str, re.Pattern[str], type]] = [ ( "mangadex.org", re.compile( @@ -108,7 +108,7 @@ class MangaDLP: self.chapter_post_hook_cmd = chapter_post_hook_cmd self.cache_path = cache_path self.add_metadata = add_metadata - self.hook_infos: dict = {} + self.hook_infos: Dict[str, Any] = {} # prepare everything self._prepare() @@ -226,7 +226,7 @@ class MangaDLP: skipped_chapters: list[Any] = [] error_chapters: list[Any] = [] for chapter in chapters_to_download: - if self.cache_path and chapter in cached_chapters: + if self.cache_path and chapter in cached_chapters: # pyright:ignore log.info(f"Chapter '{chapter}' is in cache. Skipping download") continue @@ -240,7 +240,7 @@ class MangaDLP: skipped_chapters.append(chapter) # update cache if self.cache_path: - cache.add_chapter(chapter) + cache.add_chapter(chapter) # pyright:ignore continue except Exception: # skip download/packing due to an error @@ -273,7 +273,7 @@ class MangaDLP: # update cache if self.cache_path: - cache.add_chapter(chapter) + cache.add_chapter(chapter) # pyright:ignore # start chapter post hook run_hook( @@ -310,7 +310,7 @@ class MangaDLP: # once called per chapter def get_chapter(self, chapter: str) -> Path: # get chapter infos - chapter_infos: dict = self.api.manga_chapter_data[chapter] + chapter_infos: Dict[str, Union[str, int]] = self.api.manga_chapter_data[chapter] log.debug(f"Chapter infos: {chapter_infos}") # get image urls for chapter @@ -342,8 +342,8 @@ class MangaDLP: # get filename for chapter (without suffix) chapter_filename = utils.get_filename( self.manga_title, - chapter_infos["name"], - chapter_infos["volume"], + chapter_infos["name"], # type:ignore + chapter_infos["volume"], # type:ignore chapter, self.forcevol, self.name_format, @@ -352,7 +352,7 @@ class MangaDLP: log.debug(f"Filename: '{chapter_filename}'") # set download path for chapter (image folder) - chapter_path = self.manga_path / chapter_filename + chapter_path: Path = self.manga_path / chapter_filename # set archive path with file format chapter_archive_path = Path(f"{chapter_path}{self.file_format}") diff --git a/mangadlp/cache.py b/mangadlp/cache.py index a2077b5..1781dba 100644 --- a/mangadlp/cache.py +++ b/mangadlp/cache.py @@ -26,12 +26,14 @@ class CacheDB: if not self.db_data.get(self.db_key): self.db_data[self.db_key] = {} - self.db_uuid_data: dict = self.db_data[self.db_key] + self.db_uuid_data = self.db_data[self.db_key] if not self.db_uuid_data.get("name"): self.db_uuid_data.update({"name": self.name}) self._write_db() - self.db_uuid_chapters: list = self.db_uuid_data.get("chapters") or [] + self.db_uuid_chapters: List[str] = ( + self.db_uuid_data.get("chapters") or [] # type:ignore + ) def _prepare_db(self) -> None: if self.db_path.exists(): @@ -44,11 +46,11 @@ class CacheDB: log.error("Can't create db-file") raise exc - def _read_db(self) -> Dict[str, dict]: + def _read_db(self) -> Dict[str, Dict[str, Union[str, List[str]]]]: log.info(f"Reading cache-db: {self.db_path}") try: db_txt = self.db_path.read_text(encoding="utf8") - db_dict: dict[str, dict] = json.loads(db_txt) + db_dict: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(db_txt) except Exception as exc: log.error("Can't load cache-db") raise exc @@ -73,7 +75,7 @@ class CacheDB: raise exc -def sort_chapters(chapters: list) -> List[str]: +def sort_chapters(chapters: List[str]) -> List[str]: try: sorted_list = sorted(chapters, key=float) except Exception: diff --git a/mangadlp/cli.py b/mangadlp/cli.py index 0da36a2..998e966 100644 --- a/mangadlp/cli.py +++ b/mangadlp/cli.py @@ -1,5 +1,6 @@ import sys from pathlib import Path +from typing import Any, List import click from click_option_group import ( @@ -15,7 +16,7 @@ from mangadlp.logger import prepare_logger # read in the list of links from a file -def readin_list(_ctx, _param, value) -> list: +def readin_list(_ctx: click.Context, _param: str, value: str) -> List[str]: if not value: return [] @@ -38,8 +39,8 @@ def readin_list(_ctx, _param, value) -> list: @click.help_option() @click.version_option(version=__version__, package_name="manga-dlp") # manga selection -@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup) -@optgroup.option( +@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup) # type: ignore +@optgroup.option( # type: ignore "-u", "--url", "--uuid", @@ -49,19 +50,19 @@ def readin_list(_ctx, _param, value) -> list: show_default=True, help="URL or UUID of the manga", ) -@optgroup.option( +@optgroup.option( # type: ignore "--read", "read_mangas", is_eager=True, callback=readin_list, - type=click.Path(exists=True, dir_okay=False), + type=click.Path(exists=True, dir_okay=False, path_type=str), default=None, show_default=True, help="Path of file with manga links to download. One per line", ) # logging options -@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup) -@optgroup.option( +@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup) # type: ignore +@optgroup.option( # type: ignore "--loglevel", "verbosity", type=int, @@ -69,7 +70,7 @@ def readin_list(_ctx, _param, value) -> list: show_default=True, help="Custom log level", ) -@optgroup.option( +@optgroup.option( # type: ignore "--warn", "verbosity", flag_value=30, @@ -77,7 +78,7 @@ def readin_list(_ctx, _param, value) -> list: show_default=True, help="Only log warnings and higher", ) -@optgroup.option( +@optgroup.option( # type: ignore "--debug", "verbosity", flag_value=10, @@ -227,7 +228,7 @@ def readin_list(_ctx, _param, value) -> list: help="Enable/disable creation of metadata via ComicInfo.xml", ) @click.pass_context -def main(ctx: click.Context, **kwargs) -> None: +def main(ctx: click.Context, **kwargs: Any) -> None: """Script to download mangas from various sites.""" url_uuid: str = kwargs.pop("url_uuid") read_mangas: list[str] = kwargs.pop("read_mangas") diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index 3826531..4630dd6 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -2,7 +2,7 @@ import logging import shutil from pathlib import Path from time import sleep -from typing import Union +from typing import List, Union import requests from loguru import logger as log @@ -12,7 +12,7 @@ from mangadlp import utils # download images def download_chapter( - image_urls: list, + image_urls: List[str], chapter_path: Union[str, Path], download_wait: float, ) -> None: @@ -48,8 +48,8 @@ def download_chapter( # write image try: with image_path.open("wb") as file: - r.raw.decode_content = True - shutil.copyfileobj(r.raw, file) + r.raw.decode_content = True # pyright:ignore + shutil.copyfileobj(r.raw, file) # pyright:ignore except Exception as exc: log.error("Can't write file") raise exc diff --git a/mangadlp/hooks.py b/mangadlp/hooks.py index 514af64..db3d50e 100644 --- a/mangadlp/hooks.py +++ b/mangadlp/hooks.py @@ -1,10 +1,11 @@ import os import subprocess +from typing import Any from loguru import logger as log -def run_hook(command: str, hook_type: str, **kwargs) -> int: +def run_hook(command: str, hook_type: str, **kwargs: Any) -> int: """Run a command. Run a command with subprocess.run and add kwargs to the environment. diff --git a/mangadlp/logger.py b/mangadlp/logger.py index f332ed4..f0b0897 100644 --- a/mangadlp/logger.py +++ b/mangadlp/logger.py @@ -1,5 +1,6 @@ import logging import sys +from typing import Any, Dict from loguru import logger @@ -10,7 +11,7 @@ LOGURU_FMT = "{time:%Y-%m-%dT%H:%M:%S%z} | [{level: <7}] [{name: class InterceptHandler(logging.Handler): """Intercept python logging messages and log them via loguru.logger.""" - def emit(self, record): + def emit(self, record: Any) -> None: # Get corresponding Loguru level if it exists try: level = logger.level(record.levelname).name @@ -19,8 +20,8 @@ class InterceptHandler(logging.Handler): # Find caller from where originated the logged message frame, depth = logging.currentframe(), 2 - while frame.f_code.co_filename == logging.__file__: - frame = frame.f_back + while frame.f_code.co_filename == logging.__file__: # pyright:ignore + frame = frame.f_back # type: ignore depth += 1 logger.opt(depth=depth, exception=record.exc_info).log( @@ -30,7 +31,7 @@ class InterceptHandler(logging.Handler): # init logger with format and log level def prepare_logger(loglevel: int = 20) -> None: - config: dict = { + config: Dict[str, Any] = { "handlers": [ { "sink": sys.stdout, diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index 3e31463..9ee1478 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, Tuple +from typing import Any, Dict, Tuple, Union import xmltodict from loguru import logger as log @@ -8,7 +8,7 @@ METADATA_FILENAME = "ComicInfo.xml" METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml") # define metadata types, defaults and valid values. an empty list means no value check # {key: (type, default value, valid values)} -METADATA_TYPES: Dict[str, Tuple[type, Any, list]] = { +METADATA_TYPES: Dict[str, Tuple[type, Any, list[Union[str, int]]]] = { "Title": (str, None, []), "Series": (str, None, []), "Number": (str, None, []), @@ -59,10 +59,12 @@ METADATA_TYPES: Dict[str, Tuple[type, Any, list]] = { } -def validate_metadata(metadata_in: dict) -> Dict[str, dict]: +def validate_metadata( + metadata_in: Dict[str, Union[str, int]] +) -> Dict[str, Dict[str, Union[str, int]]]: log.info("Validating metadata") - metadata_valid: dict[str, dict] = {"ComicInfo": {}} + metadata_valid: dict[str, Dict[str, Union[str, int]]] = {"ComicInfo": {}} for key, value in METADATA_TYPES.items(): metadata_type, metadata_default, metadata_validation = value @@ -104,7 +106,7 @@ def validate_metadata(metadata_in: dict) -> Dict[str, dict]: return metadata_valid -def write_metadata(chapter_path: Path, metadata: dict) -> None: +def write_metadata(chapter_path: Path, metadata: Dict[str, Union[str, int]]) -> None: if metadata["Format"] == "pdf": log.warning("Can't add metadata for pdf format. Skipping") return diff --git a/mangadlp/utils.py b/mangadlp/utils.py index d305a45..0d3c31e 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -24,7 +24,7 @@ def make_archive(chapter_path: Path, file_format: str) -> None: def make_pdf(chapter_path: Path) -> None: try: - import img2pdf # pylint: disable=import-outside-toplevel + import img2pdf # pylint: disable=import-outside-toplevel # pyright:ignore except Exception as exc: log.error("Cant import img2pdf. Please install it first") raise exc @@ -34,14 +34,14 @@ def make_pdf(chapter_path: Path) -> None: for file in chapter_path.iterdir(): images.append(str(file)) try: - pdf_path.write_bytes(img2pdf.convert(images)) + pdf_path.write_bytes(img2pdf.convert(images)) # pyright:ignore except Exception as exc: log.error("Can't create '.pdf' archive") raise exc # create a list of chapters -def get_chapter_list(chapters: str, available_chapters: list) -> List[str]: +def get_chapter_list(chapters: str, available_chapters: List[str]) -> List[str]: # check if there are available chapter chapter_list: list[str] = [] for chapter in chapters.split(","): diff --git a/pyproject.toml b/pyproject.toml index 49f557a..ac85e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ # mypy [tool.mypy] -#strict = true +strict = true python_version = "3.9" disallow_untyped_defs = false follow_imports = "normal" @@ -84,6 +84,18 @@ show_error_codes = true pretty = true no_implicit_optional = false +# pyright + +[tool.pyright] +typeCheckingMode = "strict" +pythonVersion = "3.9" +reportUnnecessaryTypeIgnoreComment = true +reportShadowedImports = true +reportUnusedExpression = true +reportMatchNotExhaustive = true +# venvPath = "." +# venv = "venv" + # ruff [tool.ruff] @@ -103,10 +115,11 @@ select = [ ] line-length = 88 fix = true +show-fixes = true format = "grouped" ignore-init-module-imports = true respect-gitignore = true -ignore = ["E501", "D103", "D100"] +ignore = ["E501", "D103", "D100", "D102", "PLR2004"] exclude = [ ".direnv", ".git", From 830cfd48bbc02ae09b00129bf9dcaae749659485 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 16:24:39 +0100 Subject: [PATCH 07/19] install deps before pyright Signed-off-by: Ivan Schaller --- .woodpecker/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml index 06097c7..f6b94ca 100644 --- a/.woodpecker/tests.yml +++ b/.woodpecker/tests.yml @@ -38,6 +38,7 @@ pipeline: image: cr.44net.ch/ci-plugins/tests pull: true commands: + - just install_deps - just test_pyright # ruff test - python From a53767bf74ca5c096306196dcd6c604f5d680f3f Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 16:29:44 +0100 Subject: [PATCH 08/19] update api template with type annotations [CI SKIP] Signed-off-by: Ivan Schaller --- contrib/api_template.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/contrib/api_template.py b/contrib/api_template.py index 89ed331..0098268 100644 --- a/contrib/api_template.py +++ b/contrib/api_template.py @@ -1,3 +1,5 @@ +from typing import List, Dict, Union + # api template for manga-dlp @@ -23,7 +25,7 @@ class YourAPI: api_base_url = "https://api.mangadex.org" img_base_url = "https://uploads.mangadex.org" - def __init__(self, url_uuid, language, forcevol): + def __init__(self, url_uuid: str, language: str, forcevol: bool): """get infos to initiate class.""" self.api_name = "Your API Name" @@ -35,7 +37,7 @@ class YourAPI: self.manga_uuid = "abc" self.manga_title = "abc" self.chapter_list = ["1", "2", "2.1", "5", "10"] - self.manga_chapter_data = { # example data + self.manga_chapter_data: Dict[str, Dict[str, str | int]] = { # example data "1": { "uuid": "abc", "volume": "1", @@ -50,7 +52,7 @@ class YourAPI: }, } # or with --forcevol - self.manga_chapter_data = { + self.manga_chapter_data: Dict[str, Dict[str, str | int]] = { "1:1": { "uuid": "abc", "volume": "1", @@ -65,7 +67,7 @@ class YourAPI: }, } - def get_chapter_images(chapter: str, download_wait: float) -> list: + def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]: """Get chapter images as a list (full links). Args: @@ -82,7 +84,7 @@ class YourAPI: "https://abc.def/image/12345.png", ] - def create_metadata(chapter: str) -> dict: + def create_metadata(self, chapter: str) -> Dict[str, Union[str, int, None]]: """Get metadata with correct keys for ComicInfo.xml. Provide as much metadata as possible. empty/false values will be ignored. @@ -95,7 +97,7 @@ class YourAPI: """ # metadata types. have to be valid # {key: (type, default value, valid values)} - { + metadata_types = { "Title": (str, None, []), "Series": (str, None, []), "Number": (str, None, []), From 32d5f8a9a11b610028a3b29209eeebcc27002e97 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sat, 18 Feb 2023 16:36:52 +0100 Subject: [PATCH 09/19] update readme [CI SKIP] Signed-off-by: Ivan Schaller --- README.md | 14 +++++++++++++- docs/pages/index.md | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bab78b9..6574a6a 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Meta [![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black) [![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff) -[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy) +[![Types](https://img.shields.io/badge/types-pyright-blue)](https://github.com/microsoft/pyright) [![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/) [![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy) [![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/) @@ -44,6 +44,18 @@ you just want the folder with all the pictures use the flag `--nocbz`. - [Mangadex.org](https://mangadex.org/) +## Features (not complete) + +- Metadata support with ComicInfo.xml +- Json caching +- Custom hooks after/before each download +- Custom chapter name format +- Volume support +- Multiple archive formats supported (cbz,cbr,zip,none) +- Language selection +- Download all chapters directly +- And others... + ## Usage ### Quick start diff --git a/docs/pages/index.md b/docs/pages/index.md index a808397..2f0d97e 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -18,7 +18,7 @@ Meta [![Code style](https://img.shields.io/badge/code%20style-black-black)](https://github.com/psf/black) [![Linter](https://img.shields.io/badge/linter-ruff-red)](https://github.com/charliermarsh/ruff) -[![Types](https://img.shields.io/badge/types-mypy-blue)](https://github.com/python/mypy) +[![Types](https://img.shields.io/badge/types-pyright-blue)](https://github.com/microsoft/pyright) [![Tests](https://img.shields.io/badge/tests-pytest%20%7C%20tox-yellow)](https://github.com/pytest-dev/pytest/) [![Coverage](https://img.shields.io/badge/coverage-coveragepy-green)](https://github.com/nedbat/coveragepy) [![License](https://img.shields.io/badge/license-MIT-9400d3.svg)](https://snyk.io/learn/what-is-mit-license/) @@ -42,6 +42,18 @@ you just want the folder with all the pictures use the flag `--nocbz`. - [Mangadex.org](https://mangadex.org/) +## Features (not complete) + +- Metadata support with ComicInfo.xml +- Json caching +- Custom hooks after/before each download +- Custom chapter name format +- Volume support +- Multiple archive formats supported (cbz,cbr,zip,none) +- Language selection +- Download all chapters directly +- And others... + ## Usage ### Quick start From 1f244ef2d661f8f2a0195f0808c39fcce82e49c9 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sun, 19 Feb 2023 18:45:53 +0100 Subject: [PATCH 10/19] remove mypy Signed-off-by: Ivan Schaller --- .woodpecker/tests.yml | 7 ------- pyproject.toml | 16 ---------------- 2 files changed, 23 deletions(-) diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml index f6b94ca..23c31b4 100644 --- a/.woodpecker/tests.yml +++ b/.woodpecker/tests.yml @@ -26,13 +26,6 @@ pipeline: commands: - just test_black - # check static typing - python - test-mypy: - image: cr.44net.ch/ci-plugins/tests - pull: true - commands: - - just test_mypy - # check static typing - python test-pyright: image: cr.44net.ch/ci-plugins/tests diff --git a/pyproject.toml b/pyproject.toml index ac85e9e..2e99004 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,22 +68,6 @@ dependencies = [ "ruff>=0.0.247", ] -# mypy - -[tool.mypy] -strict = true -python_version = "3.9" -disallow_untyped_defs = false -follow_imports = "normal" -ignore_missing_imports = true -warn_no_return = false -warn_unused_ignores = true -show_error_context = true -show_column_numbers = true -show_error_codes = true -pretty = true -no_implicit_optional = false - # pyright [tool.pyright] From e2f0a8b41f643f5f655b2020310f5fdd532a5ebf Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sun, 19 Feb 2023 18:50:56 +0100 Subject: [PATCH 11/19] fix linter paths Signed-off-by: Ivan Schaller --- contrib/api_template.py | 8 +++++--- justfile | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/api_template.py b/contrib/api_template.py index 0098268..fa2f45c 100644 --- a/contrib/api_template.py +++ b/contrib/api_template.py @@ -37,7 +37,9 @@ class YourAPI: self.manga_uuid = "abc" self.manga_title = "abc" self.chapter_list = ["1", "2", "2.1", "5", "10"] - self.manga_chapter_data: Dict[str, Dict[str, str | int]] = { # example data + self.manga_chapter_data: Dict[ + str, Dict[str, Union[str, int]] + ] = { # example data "1": { "uuid": "abc", "volume": "1", @@ -52,7 +54,7 @@ class YourAPI: }, } # or with --forcevol - self.manga_chapter_data: Dict[str, Dict[str, str | int]] = { + self.manga_chapter_data: Dict[str, Dict[str, Union[str, int]]] = { "1:1": { "uuid": "abc", "volume": "1", @@ -97,7 +99,7 @@ class YourAPI: """ # metadata types. have to be valid # {key: (type, default value, valid values)} - metadata_types = { + { "Title": (str, None, []), "Series": (str, None, []), "Number": (str, None, []), diff --git a/justfile b/justfile index d2e0b38..d33455c 100755 --- a/justfile +++ b/justfile @@ -79,7 +79,7 @@ test_shfmt: @find . -type f \( -name "**.sh" -and -not -path "./.**" -and -not -path "./venv**" \) -exec shfmt -d -i 4 -bn -ci -sr "{}" \+; test_black: - @python3 -m black --check --diff . + @python3 -m black --check --diff mangadlp/ test_mypy: @python3 -m mypy --install-types --non-interactive --ignore-missing-imports mangadlp/ @@ -88,7 +88,7 @@ test_pyright: @python3 -m pyright mangadlp/ test_ruff: - @python3 -m ruff --diff . + @python3 -m ruff --diff mangadlp/ test_ci_conf: @woodpecker-cli lint .woodpecker/ From bde2b9ebe9d002d5e4a35be0a7a1060bee029ac8 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 20 Feb 2023 14:03:40 +0100 Subject: [PATCH 12/19] add typed dicts for type hinting --- contrib/api_template.py | 12 ++++---- justfile | 6 ---- mangadlp/api/mangadex.py | 19 ++++++------ mangadlp/app.py | 8 ++--- mangadlp/metadata.py | 63 +++++++++++++++++++++++++++++++++++----- 5 files changed, 76 insertions(+), 32 deletions(-) diff --git a/contrib/api_template.py b/contrib/api_template.py index fa2f45c..dce31ff 100644 --- a/contrib/api_template.py +++ b/contrib/api_template.py @@ -1,4 +1,6 @@ -from typing import List, Dict, Union +from typing import Dict, List, Union + +from mangadlp.metadata import ChapterData # api template for manga-dlp @@ -37,20 +39,20 @@ class YourAPI: self.manga_uuid = "abc" self.manga_title = "abc" self.chapter_list = ["1", "2", "2.1", "5", "10"] - self.manga_chapter_data: Dict[ - str, Dict[str, Union[str, int]] - ] = { # example data + self.manga_chapter_data: Dict[str, ChapterData] = { # example data "1": { "uuid": "abc", "volume": "1", "chapter": "1", "name": "test", + "pages" 2, }, "2": { "uuid": "abc", "volume": "1", "chapter": "2", "name": "test", + "pages": 45, }, } # or with --forcevol @@ -86,7 +88,7 @@ class YourAPI: "https://abc.def/image/12345.png", ] - def create_metadata(self, chapter: str) -> Dict[str, Union[str, int, None]]: + def create_metadata(self, chapter: str) -> ComicInfo: """Get metadata with correct keys for ComicInfo.xml. Provide as much metadata as possible. empty/false values will be ignored. diff --git a/justfile b/justfile index d33455c..c8e5ee7 100755 --- a/justfile +++ b/justfile @@ -81,9 +81,6 @@ test_shfmt: test_black: @python3 -m black --check --diff mangadlp/ -test_mypy: - @python3 -m mypy --install-types --non-interactive --ignore-missing-imports mangadlp/ - test_pyright: @python3 -m pyright mangadlp/ @@ -120,7 +117,6 @@ lint: -just test_ci_conf just test_shfmt just test_black - just test_mypy just test_pyright just test_ruff @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" @@ -130,7 +126,6 @@ tests: -just test_ci_conf just test_shfmt just test_black - just test_mypy just test_pyright just test_ruff just test_pytest @@ -141,7 +136,6 @@ tests_full: -just test_ci_conf just test_shfmt just test_black - just test_mypy just test_pyright just test_ruff just test_build diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index 5b79460..0fc2380 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -1,11 +1,12 @@ import re from time import sleep -from typing import Any, Dict, List, Union +from typing import Any, Dict, List import requests from loguru import logger as log from mangadlp import utils +from mangadlp.metadata import ChapterData, ComicInfo class Mangadex: @@ -150,13 +151,13 @@ class Mangadex: return total_chapters # get chapter data like name, uuid etc - def get_chapter_data(self) -> Dict[str, Dict[str, Union[str, int]]]: + def get_chapter_data(self) -> Dict[str, ChapterData]: log.debug(f"Getting chapter data for: {self.manga_uuid}") api_sorting = "order[chapter]=asc&order[volume]=asc" # check for chapters in specified lang total_chapters = self.check_chapter_lang() - chapter_data = {} + chapter_data: dict[str, ChapterData] = {} last_volume, last_chapter = ("", "") offset = 0 while offset < total_chapters: # if more than 500 chapters @@ -205,7 +206,7 @@ class Mangadex: # increase offset for mangas with more than 500 chapters offset += 500 - return chapter_data # type:ignore + return chapter_data # get images for the chapter (mangadex@home) def get_chapter_images(self, chapter: str, wait_time: float) -> List[str]: @@ -259,8 +260,8 @@ class Mangadex: log.debug(f"Creating chapter list for: {self.manga_uuid}") chapter_list: List[str] = [] for data in self.manga_chapter_data.values(): - chapter_number: str = data["chapter"] # type:ignore - volume_number: str = data["volume"] # type:ignore + chapter_number: str = data["chapter"] + volume_number: str = data["volume"] if self.forcevol: chapter_list.append(f"{volume_number}:{chapter_number}") else: @@ -268,7 +269,7 @@ class Mangadex: return chapter_list - def create_metadata(self, chapter: str) -> Dict[str, Union[str, int, None]]: + def create_metadata(self, chapter: str) -> ComicInfo: log.info("Creating metadata from api") chapter_data = self.manga_chapter_data[chapter] @@ -276,7 +277,7 @@ class Mangadex: volume = int(chapter_data["volume"]) except (ValueError, TypeError): volume = None - metadata = { + metadata: ComicInfo = { "Volume": volume, "Number": chapter_data.get("chapter"), "PageCount": chapter_data.get("pages"), @@ -289,4 +290,4 @@ class Mangadex: "Web": f"https://mangadex.org/title/{self.manga_uuid}", } - return metadata # pyright:ignore + return metadata diff --git a/mangadlp/app.py b/mangadlp/app.py index 4b8ad91..7aa7fec 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -9,7 +9,7 @@ from mangadlp import downloader, utils from mangadlp.api.mangadex import Mangadex from mangadlp.cache import CacheDB from mangadlp.hooks import run_hook -from mangadlp.metadata import write_metadata +from mangadlp.metadata import ChapterData, write_metadata from mangadlp.utils import get_file_format @@ -310,7 +310,7 @@ class MangaDLP: # once called per chapter def get_chapter(self, chapter: str) -> Path: # get chapter infos - chapter_infos: Dict[str, Union[str, int]] = self.api.manga_chapter_data[chapter] + chapter_infos: ChapterData = self.api.manga_chapter_data[chapter] log.debug(f"Chapter infos: {chapter_infos}") # get image urls for chapter @@ -342,8 +342,8 @@ class MangaDLP: # get filename for chapter (without suffix) chapter_filename = utils.get_filename( self.manga_title, - chapter_infos["name"], # type:ignore - chapter_infos["volume"], # type:ignore + chapter_infos["name"], + chapter_infos["volume"], chapter, self.forcevol, self.name_format, diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index 9ee1478..78eea57 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, Tuple, Union +from typing import Any, Dict, Optional, Tuple, TypedDict, Union import xmltodict from loguru import logger as log @@ -59,12 +59,59 @@ METADATA_TYPES: Dict[str, Tuple[type, Any, list[Union[str, int]]]] = { } -def validate_metadata( - metadata_in: Dict[str, Union[str, int]] -) -> Dict[str, Dict[str, Union[str, int]]]: +class ComicInfo(TypedDict, total=False): + """ComicInfo.xml basic types. + + Validation is done via metadata.validate_metadata() + All valid types and values are specified in metadata.METADATA_TYPES + """ + + Title: Optional[str] + Series: Optional[str] + Number: Optional[str] + Count: Optional[int] + Volume: Optional[int] + AlternateSeries: Optional[str] + AlternateNumber: Optional[str] + AlternateCount: Optional[int] + Summary: Optional[str] + Notes: Optional[str] + Year: Optional[int] + Month: Optional[int] + Day: Optional[int] + Writer: Optional[str] + Colorist: Optional[str] + Publisher: Optional[str] + Genre: Optional[str] + Web: Optional[str] + PageCount: Optional[int] + LanguageISO: Optional[str] + Format: Optional[str] + BlackAndWhite: Optional[str] + Manga: Optional[str] + ScanInformation: Optional[str] + SeriesGroup: Optional[str] + AgeRating: Optional[str] + CommunityRating: Optional[int] + + +class ChapterData(TypedDict): + """Basic chapter-data types. + + All values have to be provided. + """ + + uuid: str + volume: str + chapter: str + name: str + pages: int + + +def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]: log.info("Validating metadata") - metadata_valid: dict[str, Dict[str, Union[str, int]]] = {"ComicInfo": {}} + metadata_valid: dict[str, ComicInfo] = {"ComicInfo": {}} for key, value in METADATA_TYPES.items(): metadata_type, metadata_default, metadata_validation = value @@ -77,7 +124,7 @@ def validate_metadata( # check if metadata key is available try: - md_to_check = metadata_in[key] + md_to_check: Union[str, int, None] = metadata_in[key] except KeyError: continue # check if provided metadata item is empty @@ -106,8 +153,8 @@ def validate_metadata( return metadata_valid -def write_metadata(chapter_path: Path, metadata: Dict[str, Union[str, int]]) -> None: - if metadata["Format"] == "pdf": +def write_metadata(chapter_path: Path, metadata: ComicInfo) -> None: + if metadata["Format"] == "pdf": # pyright:ignore log.warning("Can't add metadata for pdf format. Skipping") return From 9935c97f6cc63592727ab9608ebae216d86f2567 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Mon, 20 Feb 2023 14:38:09 +0100 Subject: [PATCH 13/19] move custom types to own file --- contrib/api_template.py | 6 ++--- mangadlp/api/mangadex.py | 2 +- mangadlp/app.py | 3 ++- mangadlp/metadata.py | 53 +++------------------------------------- mangadlp/types.py | 50 +++++++++++++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 55 deletions(-) create mode 100644 mangadlp/types.py diff --git a/contrib/api_template.py b/contrib/api_template.py index dce31ff..ea02391 100644 --- a/contrib/api_template.py +++ b/contrib/api_template.py @@ -1,6 +1,6 @@ from typing import Dict, List, Union -from mangadlp.metadata import ChapterData +from mangadlp.types import ChapterData,ComicInfo # api template for manga-dlp @@ -56,7 +56,7 @@ class YourAPI: }, } # or with --forcevol - self.manga_chapter_data: Dict[str, Dict[str, Union[str, int]]] = { + self.manga_chapter_data: Dict[str, ChapterData] = { "1:1": { "uuid": "abc", "volume": "1", @@ -157,7 +157,7 @@ class YourAPI: # example return { - "Volume": "abc", + "Volume": 1, "LanguageISO": "en", "Title": "test", } diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index 0fc2380..1d7b046 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -6,7 +6,7 @@ import requests from loguru import logger as log from mangadlp import utils -from mangadlp.metadata import ChapterData, ComicInfo +from mangadlp.types import ChapterData, ComicInfo class Mangadex: diff --git a/mangadlp/app.py b/mangadlp/app.py index 7aa7fec..6ee44c0 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -9,7 +9,8 @@ from mangadlp import downloader, utils from mangadlp.api.mangadex import Mangadex from mangadlp.cache import CacheDB from mangadlp.hooks import run_hook -from mangadlp.metadata import ChapterData, write_metadata +from mangadlp.metadata import write_metadata +from mangadlp.types import ChapterData from mangadlp.utils import get_file_format diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index 78eea57..d183147 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -1,9 +1,11 @@ from pathlib import Path -from typing import Any, Dict, Optional, Tuple, TypedDict, Union +from typing import Any, Dict, Tuple, Union import xmltodict from loguru import logger as log +from mangadlp.types import ComicInfo + METADATA_FILENAME = "ComicInfo.xml" METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml") # define metadata types, defaults and valid values. an empty list means no value check @@ -59,55 +61,6 @@ METADATA_TYPES: Dict[str, Tuple[type, Any, list[Union[str, int]]]] = { } -class ComicInfo(TypedDict, total=False): - """ComicInfo.xml basic types. - - Validation is done via metadata.validate_metadata() - All valid types and values are specified in metadata.METADATA_TYPES - """ - - Title: Optional[str] - Series: Optional[str] - Number: Optional[str] - Count: Optional[int] - Volume: Optional[int] - AlternateSeries: Optional[str] - AlternateNumber: Optional[str] - AlternateCount: Optional[int] - Summary: Optional[str] - Notes: Optional[str] - Year: Optional[int] - Month: Optional[int] - Day: Optional[int] - Writer: Optional[str] - Colorist: Optional[str] - Publisher: Optional[str] - Genre: Optional[str] - Web: Optional[str] - PageCount: Optional[int] - LanguageISO: Optional[str] - Format: Optional[str] - BlackAndWhite: Optional[str] - Manga: Optional[str] - ScanInformation: Optional[str] - SeriesGroup: Optional[str] - AgeRating: Optional[str] - CommunityRating: Optional[int] - - -class ChapterData(TypedDict): - """Basic chapter-data types. - - All values have to be provided. - """ - - uuid: str - volume: str - chapter: str - name: str - pages: int - - def validate_metadata(metadata_in: ComicInfo) -> Dict[str, ComicInfo]: log.info("Validating metadata") diff --git a/mangadlp/types.py b/mangadlp/types.py new file mode 100644 index 0000000..b924bf4 --- /dev/null +++ b/mangadlp/types.py @@ -0,0 +1,50 @@ +from typing import Optional, TypedDict + + +class ComicInfo(TypedDict, total=False): + """ComicInfo.xml basic types. + + Validation is done via metadata.validate_metadata() + All valid types and values are specified in metadata.METADATA_TYPES + """ + + Title: Optional[str] + Series: Optional[str] + Number: Optional[str] + Count: Optional[int] + Volume: Optional[int] + AlternateSeries: Optional[str] + AlternateNumber: Optional[str] + AlternateCount: Optional[int] + Summary: Optional[str] + Notes: Optional[str] + Year: Optional[int] + Month: Optional[int] + Day: Optional[int] + Writer: Optional[str] + Colorist: Optional[str] + Publisher: Optional[str] + Genre: Optional[str] + Web: Optional[str] + PageCount: Optional[int] + LanguageISO: Optional[str] + Format: Optional[str] + BlackAndWhite: Optional[str] + Manga: Optional[str] + ScanInformation: Optional[str] + SeriesGroup: Optional[str] + AgeRating: Optional[str] + CommunityRating: Optional[int] + + +class ChapterData(TypedDict): + """Basic chapter-data types. + + All values have to be provided. + """ + + uuid: str + volume: str + chapter: str + name: str + pages: int From 0f9e718e308f48f8bf96a49bac33e3177637d6f1 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Tue, 21 Feb 2023 17:16:39 +0100 Subject: [PATCH 14/19] update readme Signed-off-by: Ivan Schaller --- README.md | 12 ++++++------ docs/pages/index.md | 18 ++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 6574a6a..ee291c9 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,14 @@ Meta ## Description A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support -for other sites is planned. +for other sites is _planned™_. Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new chapters without any additional setup. The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If -you just want the folder with all the pictures use the flag `--nocbz`. +you just want the folder with all the pictures use the flag `--format ""`. ## _Currently_ Supported sites @@ -137,17 +137,17 @@ verbosity: [mutually_exclusive] For suggestions for improvement, just open a pull request. If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use. -And more infos and tools in the contrib [README.md](contrib/README.md) +And more infos and tools are in the contrib [README.md](contrib/README.md) -Otherwise, you can open am issue with the name of the site which you want support for. (not guaranteed to be -implemented) +Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be +implemented). If you encounter any bugs, also just open an issue with a description of the problem. ## TODO's - Make docker container for easy distribution - --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp) + --> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp) - Automate release --> Done with woodpecker-ci - Make pypi package diff --git a/docs/pages/index.md b/docs/pages/index.md index 2f0d97e..e176827 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -29,14 +29,14 @@ Meta ## Description A manga download script written in python. It only supports [mangadex.org](https://mangadex.org/) for now. But support -for other sites is planned. +for other sites is _planned™_. Before downloading a new chapter, the script always checks if there is already a chapter with the same name in the download directory. If found the chapter is skipped. So you can run the script on a schedule to only download new chapters without any additional setup. The default behaiviour is to pack the images to a [cbz archive](https://en.wikipedia.org/wiki/Comic_book_archive). If -you just want the folder with all the pictures use the flag `--nocbz`. +you just want the folder with all the pictures use the flag `--format ""`. ## _Currently_ Supported sites @@ -94,7 +94,7 @@ mangadlp # call script directly ### With docker -See the docker [README](docker/) +See the docker [README](https://manga-dlp.ivn.sh/docker/) ## Options @@ -134,20 +134,18 @@ verbosity: [mutually_exclusive] For suggestions for improvement, just open a pull request. -If you want to add support for a new site, there is an -api [template file](https://github.com/olofvndrhr/manga-dlp/blob/master/contrib/api_template.py) which you can use. -And more infos and tools in the -contrib [README.md](https://github.com/olofvndrhr/manga-dlp/blob/master/contrib/README.md) +If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use. +And more infos and tools are in the contrib [README.md](contrib/README.md) -Otherwise, you can open am issue with the name of the site which you want support for. (not guaranteed to be -implemented) +Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be +implemented). If you encounter any bugs, also just open an issue with a description of the problem. ## TODO's - Make docker container for easy distribution - --> [Dockerhub](https://hub.docker.com/repository/docker/olofvndrhr/manga-dlp) + --> [Dockerhub](https://hub.docker.com/r/olofvndrhr/manga-dlp) - Automate release --> Done with woodpecker-ci - Make pypi package From 3d5186966318129b304b7f9b5315be81eeed1454 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Wed, 22 Feb 2023 21:05:25 +0100 Subject: [PATCH 15/19] update CHANGELOG and bump version Signed-off-by: Ivan Schaller --- CHANGELOG.md | 362 ++++++++++++++++++++++-------------------- mangadlp/__about__.py | 2 +- 2 files changed, 190 insertions(+), 174 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53a88c7..6ea90f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,442 +7,458 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -- Add support for more sites +- Add support for more sites + +## [2.3.1] - 2023-02-XX + +### Added + +- Added TypedDicts for type checkers and type annotation + +### Fixed + +- Fixed some typos in the README + +### Changed + +- Switched from pylint/pylama/isort/autoflake to ruff +- Switched from mypy to pyright and added strict type checking +- Updated the api template ## [2.3.0] - 2023-02-15 ### Added -- Metadata is now added to each chapter. Schema - standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0) -- Added `xmltodict` as a package requirement -- Cache now also saves the manga title -- New tests -- More typo annotations for function, compatible with python3.8 -- File format checker if you use the MangaDLP class directly +- Metadata is now added to each chapter. Schema + standard: [https://anansi-project.github.io/docs/comicinfo/schemas/v2.0](https://anansi-project.github.io/docs/comicinfo/schemas/v2.0) +- Added `xmltodict` as a package requirement +- Cache now also saves the manga title +- New tests +- More typo annotations for function, compatible with python3.8 +- File format checker if you use the MangaDLP class directly ### Fixed -- API template typos -- Some useless type annotations +- API template typos +- Some useless type annotations ### Changed -- Simplified the chapter info generation -- Updated the license year -- Updated the API template -- Updated the API detection and removed it from the MangaDLP class +- Simplified the chapter info generation +- Updated the license year +- Updated the API template +- Updated the API detection and removed it from the MangaDLP class ## [2.2.20] - 2023-02-12 ### Fixed -- Script now doesn't exit if multiple mangas were requested and one had an error +- Script now doesn't exit if multiple mangas were requested and one had an error ## [2.2.19] - 2023-02-11 ### Added -- First version of the chapter cache (very basic functionality) +- First version of the chapter cache (very basic functionality) ### Fixed -- Fixed all exception re-raises to include the original stack trace +- Fixed all exception re-raises to include the original stack trace ### Changed -- Simplified chapter download loop +- Simplified chapter download loop ## [2.2.18] - 2023-01-21 ### Fixed -- Fixed manga titles on non english language -- Fixed title & filename fixing to not use `ascii` but `uft8` +- Fixed manga titles on non english language +- Fixed title & filename fixing to not use `ascii` but `uft8` ### Added -- Fallback title to english of none was found in requested language -- More debug logs -- More tests +- Fallback title to english of none was found in requested language +- More debug logs +- More tests ### Changed -- Now uses the first found alt-title. Before it was the last -- Removed `sys.exit` in the api +- Now uses the first found alt-title. Before it was the last +- Removed `sys.exit` in the api ## [2.2.17] - 2023-01-15 ### Fixed -- Set a timeout of 10 seconds for the api requests +- Set a timeout of 10 seconds for the api requests ### Added -- `--name-format` and `--name-format-none` flags to add a custom naming scheme for the downloaded files. See - docs: https://manga-dlp.ivn.sh/download/ -- More debug log messages -- More tests for the custom naming scheme -- More type hints +- `--name-format` and `--name-format-none` flags to add a custom naming scheme for the downloaded files. See + docs: https://manga-dlp.ivn.sh/download/ +- More debug log messages +- More tests for the custom naming scheme +- More type hints ### Changed -- Make `--format` a `click.Choice` option -- In the `--format` option the leading dot is now invalid. `--format .cbz` -> `--format cbz` -- Changed empty values from the api from None to an empty string -- Minor code readability improvements +- Make `--format` a `click.Choice` option +- In the `--format` option the leading dot is now invalid. `--format .cbz` -> `--format cbz` +- Changed empty values from the api from None to an empty string +- Minor code readability improvements ## [2.2.16] - 2022-12-30 ### Fixed -- Log level is now fixed and should not default to 0 -- Docker schedule should now work again +- Log level is now fixed and should not default to 0 +- Docker schedule should now work again ### Changed -- Integrate logging logs to loguru via custom sink -- Simplify docker shell scripts +- Integrate logging logs to loguru via custom sink +- Simplify docker shell scripts ## [2.2.15] - 2022-12-29 ### Added -- `--warn` and `--loglevel` flags +- `--warn` and `--loglevel` flags ### Removed -- Remove `--lean` and `--verbose` flags and remove custom log levels +- Remove `--lean` and `--verbose` flags and remove custom log levels ### Changed -- Move from standard library logging to [loguru](https://loguru.readthedocs.io/en/stable/index.html) -- Move from standard library argparse to [click](https://click.palletsprojects.com/en/8.1.x/) +- Move from standard library logging to [loguru](https://loguru.readthedocs.io/en/stable/index.html) +- Move from standard library argparse to [click](https://click.palletsprojects.com/en/8.1.x/) ## [2.2.14] - 2022-10-06 ### Changed -- Changed logging format to ISO 8601 -- Small logging corrections +- Changed logging format to ISO 8601 +- Small logging corrections ## [2.2.13] - 2022-08-15 ### Added -- Option to run custom hooks before and after each chapter/manga download -- _Tests for the new hooks_ -- _Docs for the new hooks_ -- _Tests for mkdocs generation_ +- Option to run custom hooks before and after each chapter/manga download +- _Tests for the new hooks_ +- _Docs for the new hooks_ +- _Tests for mkdocs generation_ ### Changed -- Verbose and Debug logging now have a space as a seperator between log level-name and log-level -- APIs now have an attribute with their name (for the hooks) - `api.api_name` -- Docs moved to Cloudflare pages (generated with mkdocs) +- Verbose and Debug logging now have a space as a seperator between log level-name and log-level +- APIs now have an attribute with their name (for the hooks) - `api.api_name` +- Docs moved to Cloudflare pages (generated with mkdocs) ## [2.1.12] - 2022-07-25 ### Fixed -- Image publishing with `hatch` on pypi should now work again -- The schedule fixer for the new `.sh` schedule should now work correctly +- Image publishing with `hatch` on pypi should now work again +- The schedule fixer for the new `.sh` schedule should now work correctly ### Added -- More CI tests: `pylint`, `pylama` and `autoflake` -- New function in `get_release_notes.sh` to get the latest version -- Docstrings for `MangaDLP` and the api module `Mangadex` +- More CI tests: `pylint`, `pylama` and `autoflake` +- New function in `get_release_notes.sh` to get the latest version +- Docstrings for `MangaDLP` and the api module `Mangadex` ### Changed -- CI workflow is now faster and runs natively on arm64 (before it was buildx/emulation) -- `Pylint`/`pylama` code improvements -- Version management is now done with `hatch` (in `__about__.py`) +- CI workflow is now faster and runs natively on arm64 (before it was buildx/emulation) +- `Pylint`/`pylama` code improvements +- Version management is now done with `hatch` (in `__about__.py`) ## [2.1.11] - 2022-07-18 ### Fixed -- The `--read` option now filters empty lines, so it will not generate an error anymore -- An error which was caused by the interactive input method when you did not specify a chapter or to list them -- Some typos +- The `--read` option now filters empty lines, so it will not generate an error anymore +- An error which was caused by the interactive input method when you did not specify a chapter or to list them +- Some typos ### Added -- Options to configure the default schedule in the docker container via environment variables -- Section the the docker [README.md](docker/README.md) for the new environment variables -- `autoflake` test in `justfile` -- Some more things which get logged +- Options to configure the default schedule in the docker container via environment variables +- Section the the docker [README.md](docker/README.md) for the new environment variables +- `autoflake` test in `justfile` +- Some more things which get logged ### Changed -- **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to - overwrite - the default schedule -- Added the `.sh` suffix to the s6 init scripts for better compatibility -- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other - improvements +- **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to + overwrite + the default schedule +- Added the `.sh` suffix to the s6 init scripts for better compatibility +- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other + improvements ## [2.1.10] - 2022-07-14 ### Fixed -- Removed some unused files +- Removed some unused files ### Added -- `logger.py` for all log related settings and functions +- `logger.py` for all log related settings and functions ### Changed -- Logging of output. The script now uses the `logging` library +- Logging of output. The script now uses the `logging` library ## [2.1.9] - 2022-06-26 ### Fixed -- Timeouts in tests, due to api limitations. Now added a wait time between tests -- Pytest path +- Timeouts in tests, due to api limitations. Now added a wait time between tests +- Pytest path ### Added -- `--lean` flag for less output -- [justfile](https://github.com/casey/just) for setting up a dev environment and testing the code -- [asdf](https://github.com/asdf-vm/asdf) for version management -- Dev requirements in [contrib/requirements_dev.txt](contrib/requirements_dev.txt) -- `README` in [contrib](contrib) +- `--lean` flag for less output +- [justfile](https://github.com/casey/just) for setting up a dev environment and testing the code +- [asdf](https://github.com/asdf-vm/asdf) for version management +- Dev requirements in [contrib/requirements_dev.txt](contrib/requirements_dev.txt) +- `README` in [contrib](contrib) ### Changed -- Handling of verbosity and logging. Now there are 4 types of verbosity: `normal`, `lean`, `verbose` and `debug` -- CI/CD pipeline for testing and releases -- Coverage testing now also done with `tox` -- Default verbosity of docker container is now `--lean` -- Reorganised [pyproject.toml](pyproject.toml) +- Handling of verbosity and logging. Now there are 4 types of verbosity: `normal`, `lean`, `verbose` and `debug` +- CI/CD pipeline for testing and releases +- Coverage testing now also done with `tox` +- Default verbosity of docker container is now `--lean` +- Reorganised [pyproject.toml](pyproject.toml) ## [2.1.8] - 2022-06-22 ### Fixed -- Interactive input +- Interactive input ## [2.1.7] - 2022-06-22 ### Added -- tox version testing -- New pre-release tests -- Build info's with hatch -- [Pypi](https://pypi.org/project/manga-dlp/) build with hatch -- Pypi section in `README.md` -- [Snyk](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3) test results - in `README.md` +- tox version testing +- New pre-release tests +- Build info's with hatch +- [Pypi](https://pypi.org/project/manga-dlp/) build with hatch +- Pypi section in `README.md` +- [Snyk](https://app.snyk.io/org/olofvndrhr-t6h/project/aae9609d-a4e4-41f8-b1ac-f2561b2ad4e3) test results + in `README.md` ### Changed -- Moved code from `manga-dlp.py` to `input.py` for uniformity -- The default entrypoint is now `mangadlp.input:main` +- Moved code from `manga-dlp.py` to `input.py` for uniformity +- The default entrypoint is now `mangadlp.input:main` ## [2.1.6] - 2022-06-21 ### Fixed -- Docker labels are now working -- Global variables are now fully uppercase -- Some errors with static types +- Docker labels are now working +- Global variables are now fully uppercase +- Some errors with static types ### Added -- bump2version config for releases -- More tests with: `mypy` and `isort` -- New issue templates +- bump2version config for releases +- More tests with: `mypy` and `isort` +- New issue templates ### Changed -- Release workflow now is based on configuration files -- Switched from `setup.py` to `pyproject.toml` -- `README.md` now has sorted badges -- Imports are now sorted with `isort` -- Static types are now checked with `mypy` -- Release note generation is now simplified +- Release workflow now is based on configuration files +- Switched from `setup.py` to `pyproject.toml` +- `README.md` now has sorted badges +- Imports are now sorted with `isort` +- Static types are now checked with `mypy` +- Release note generation is now simplified ## [2.1.5] - 2022-06-18 ### Fixed -- Image names now have a suffix, as some comic readers have problems with no - suffix [fixes issue #2] +- Image names now have a suffix, as some comic readers have problems with no + suffix [fixes issue #2] ### Added -- `--format` section in the README +- `--format` section in the README ## [2.1.4] - 2022-05-29 ### Fixed -- Docker container now works again -- Fixed cron in docker container +- Docker container now works again +- Fixed cron in docker container ### Changed -- Docker container scheduling is now more practical +- Docker container scheduling is now more practical ## [2.1.3] - 2022-05-29 ### Fixed -- Error-chapters and skipped-chapters list are now shown again -- The Interactive input version now matches `--version` +- Error-chapters and skipped-chapters list are now shown again +- The Interactive input version now matches `--version` ### Added -- Ability to list chapters with interactive input +- Ability to list chapters with interactive input ### Changed -- Replace `exit()` with `sys.exit()` -- Renamed class methods to not look like dunder methods -- Script execution moved from `os.system()` to `subprocess.call()` +- Replace `exit()` with `sys.exit()` +- Renamed class methods to not look like dunder methods +- Script execution moved from `os.system()` to `subprocess.call()` ## [2.1.2] - 2022-05-20 ### Fixed -- List chapters when none were specified -- Typos +- List chapters when none were specified +- Typos ### Added -- Ability to download whole volumes +- Ability to download whole volumes ### Changed -- Moved processing of list with links to input.py -- Updated README for volume and chapter selection +- Moved processing of list with links to input.py +- Updated README for volume and chapter selection ## [2.1.1] - 2022-05-18 ### Fixed -- Progress bar on verbose output -- Sonarqube link for CI -- A few typos -- Removed unnecessary escapes from file rename regex +- Progress bar on verbose output +- Sonarqube link for CI +- A few typos +- Removed unnecessary escapes from file rename regex ### Added -- API template +- API template ### Changed -- Updated docker baseimage -- Rewrote app.py to a class +- Updated docker baseimage +- Rewrote app.py to a class ## [2.1.0] - 2022-05-16 ### Fixed -- Detection of files. Now it will skip them again +- Detection of files. Now it will skip them again ### Added -- Ability to save the chapters as pdf (only on amd64/x86) -- New output formats: rar, zip -- Progress bar to show image download -- Interactive input if no command line flags are given -- Better KeyboardInterrupt handling -- Better error handling -- Removed duplicate code +- Ability to save the chapters as pdf (only on amd64/x86) +- New output formats: rar, zip +- Progress bar to show image download +- Interactive input if no command line flags are given +- Better KeyboardInterrupt handling +- Better error handling +- Removed duplicate code ### Changed -- How the variables are used inside the script -- Variables have now the same name as in other scripts (mostly) -- Better retrying when a task fails +- How the variables are used inside the script +- Variables have now the same name as in other scripts (mostly) +- Better retrying when a task fails ## [2.0.8] - 2022-05-13 ### Changed -- Rewrote parts of script to be easier to maintain -- Moved the input script to the base folder -- Moved all arguments to a class -- Docker container creation +- Rewrote parts of script to be easier to maintain +- Moved the input script to the base folder +- Moved all arguments to a class +- Docker container creation ## [2.0.7] - 2022-05-13 ### Changed -- Changed CI/CD Platform from Drone-CI to Woodpecker-CI -- Release title is now only the version +- Changed CI/CD Platform from Drone-CI to Woodpecker-CI +- Release title is now only the version ## [2.0.6] - 2022-05-11 ### Fixed -- Filenames on windows (ntfs). Removed double quote from file and folder names +- Filenames on windows (ntfs). Removed double quote from file and folder names ## [2.0.5] - 2022-05-11 ### Fixed -- Better error handling on "KeyboardInterrupt" -- Release notes now fixed +- Better error handling on "KeyboardInterrupt" +- Release notes now fixed ### Added -- New test cases +- New test cases ## [2.0.4] - 2022-05-10 ### Added -- New test cases for more coverage -- Github release -- Updated docker baseimage +- New test cases for more coverage +- Github release +- Updated docker baseimage ## [2.0.3] - 2022-05-10 ### Fixed -- Test cases now work again -- Sonarqube settings +- Test cases now work again +- Sonarqube settings ### Added -- Coverage report in sonarqube -- Gitea release +- Coverage report in sonarqube +- Gitea release ## [2.0.2] - 2022-05-09 ### Fixed -- Restart failed api requests -- Added wait time for image gathering, as to stop api rate limiting from mangadex -- "--wait" options now works properly again +- Restart failed api requests +- Added wait time for image gathering, as to stop api rate limiting from mangadex +- "--wait" options now works properly again ## [2.0.1] - 2022-05-09 ### Fixed -- Regex for removing illegal characters in the filenames now doesn't remove quotes -- Updated docker baseimage and fixed the mangadlp tag in it -- Update license for 2022 +- Regex for removing illegal characters in the filenames now doesn't remove quotes +- Updated docker baseimage and fixed the mangadlp tag in it +- Update license for 2022 ### Added -- Quick start section in README -- Preperation for pypi +- Quick start section in README +- Preperation for pypi ## [2.0.0] - 2022-05-09 ### Fixed -- Support for new mangadex api +- Support for new mangadex api ### Changed -- Code is now formatted with [black](https://github.com/psf/black) -- Now also supports just the uuid for managex (not a full link) +- Code is now formatted with [black](https://github.com/psf/black) +- Now also supports just the uuid for managex (not a full link) diff --git a/mangadlp/__about__.py b/mangadlp/__about__.py index 55e4709..3a5935a 100644 --- a/mangadlp/__about__.py +++ b/mangadlp/__about__.py @@ -1 +1 @@ -__version__ = "2.3.0" +__version__ = "2.3.1" From 0ada98529ab72fb12d8dbf41976d7506277693da Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Wed, 22 Feb 2023 21:41:23 +0100 Subject: [PATCH 16/19] update readme links Signed-off-by: Ivan Schaller --- README.md | 4 ++-- docs/pages/index.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ee291c9..84e1950 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ you just want the folder with all the pictures use the flag `--format ""`. ## Features (not complete) -- Metadata support with ComicInfo.xml +- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro) - Json caching - Custom hooks after/before each download - Custom chapter name format @@ -136,7 +136,7 @@ verbosity: [mutually_exclusive] For suggestions for improvement, just open a pull request. -If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use. +If you want to add support for a new site, there is an api [template file](contrib/api_template.py) which you can use. And more infos and tools are in the contrib [README.md](contrib/README.md) Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be diff --git a/docs/pages/index.md b/docs/pages/index.md index e176827..e7cac2d 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -44,7 +44,7 @@ you just want the folder with all the pictures use the flag `--format ""`. ## Features (not complete) -- Metadata support with ComicInfo.xml +- Metadata support with [ComicInfo.xml](https://anansi-project.github.io/docs/comicinfo/intro) - Json caching - Custom hooks after/before each download - Custom chapter name format @@ -134,8 +134,8 @@ verbosity: [mutually_exclusive] For suggestions for improvement, just open a pull request. -If you want to add support for a new site, there is an api [template file](./contrib/api_template.py) which you can use. -And more infos and tools are in the contrib [README.md](contrib/README.md) +If you want to add support for a new site, there is an api [template file](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/api_template.py) which you can use. +And more infos and tools are in the contrib [README.md](https://github.com/olofvndrhr/manga-dlp/tree/master/contrib/README.md) Otherwise, you can open an issue with the name of the site which you want support for (not guaranteed to be implemented). From 987f72715cd70c68e05bb5b21792b2c11bf96628 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sun, 12 Mar 2023 00:32:37 +0100 Subject: [PATCH 17/19] update release date [CI SKIP] Signed-off-by: Ivan Schaller --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ea90f5..a06ac4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Add support for more sites -## [2.3.1] - 2023-02-XX +## [2.3.1] - 2023-03-12 ### Added From f7eebc2deca51bc9908a2ed61f4e27081339b86c Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sun, 12 Mar 2023 01:04:37 +0100 Subject: [PATCH 18/19] fix type hint for py38 Signed-off-by: Ivan Schaller --- mangadlp/metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index d183147..f282cc7 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -10,7 +10,9 @@ METADATA_FILENAME = "ComicInfo.xml" METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml") # define metadata types, defaults and valid values. an empty list means no value check # {key: (type, default value, valid values)} -METADATA_TYPES: Dict[str, Tuple[type, Any, list[Union[str, int]]]] = { +METADATA_TYPES: Dict[ + str, Tuple[Any, Union[str, int, None], list[Union[str, int, None]]] +] = { "Title": (str, None, []), "Series": (str, None, []), "Number": (str, None, []), From 8173b2a729ba56ad7187b9a7402d9b4d4247cbe5 Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Sun, 12 Mar 2023 01:51:51 +0100 Subject: [PATCH 19/19] fix type annitation for py38 Signed-off-by: Ivan Schaller --- mangadlp/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mangadlp/metadata.py b/mangadlp/metadata.py index f282cc7..2bd9701 100644 --- a/mangadlp/metadata.py +++ b/mangadlp/metadata.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, Tuple, Union +from typing import Any, Dict, List, Tuple, Union import xmltodict from loguru import logger as log @@ -11,7 +11,7 @@ METADATA_TEMPLATE = Path("mangadlp/metadata/ComicInfo_v2.0.xml") # define metadata types, defaults and valid values. an empty list means no value check # {key: (type, default value, valid values)} METADATA_TYPES: Dict[ - str, Tuple[Any, Union[str, int, None], list[Union[str, int, None]]] + str, Tuple[Any, Union[str, int, None], List[Union[str, int, None]]] ] = { "Title": (str, None, []), "Series": (str, None, []),