From 820c891fd7821c35acbd68fb1a0a79502ccdeebe Mon Sep 17 00:00:00 2001 From: Ivan Schaller Date: Fri, 15 Jul 2022 14:04:22 +0200 Subject: [PATCH] logging improvements --- CHANGELOG.md | 17 ++++++++++++++++ justfile | 6 ++++++ mangadlp/__init__.py | 9 ++------- mangadlp/api/mangadex.py | 18 +++++++++-------- mangadlp/app.py | 41 ++++++++++++++++----------------------- mangadlp/downloader.py | 5 +++-- mangadlp/input.py | 20 +++++++++++++------ mangadlp/logger.py | 42 ++++++++++++++++++++++++++++++++++++++++ mangadlp/utils.py | 5 +++-- tests/test_01_app.py | 5 +++-- tests/test_02_utils.py | 1 - tests/test_21_full.py | 1 - 12 files changed, 117 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 196e60a..ef2fa84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,23 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Add support for more sites +## [2.1.11] - 2022-07- + +### Fixed + +- The `--read` option now filters empty lines, so it will not generate an error anymore +- An error which was caused by the interactive input method when you did not specify a chapter or to list them + +### Added + +- `autoflake` test in `justfile` +- Some more things which get logged + +### Changed + +- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other + improvements + ## [2.1.10] - 2022-07-14 ### Fixed diff --git a/justfile b/justfile index 72d91af..c199ea6 100755 --- a/justfile +++ b/justfile @@ -82,6 +82,10 @@ test_mypy: test_pytest: @python3 -m tox -e basic +test_autoflake: + @python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/ + @python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/ + test_tox: @python3 -m tox @@ -111,6 +115,7 @@ lint: just test_black just test_isort just test_mypy + just test_autoflake @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" tests: @@ -120,6 +125,7 @@ tests: just test_black just test_isort just test_mypy + just test_autoflake just test_pytest @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" diff --git a/mangadlp/__init__.py b/mangadlp/__init__.py index 5063518..c2a631f 100644 --- a/mangadlp/__init__.py +++ b/mangadlp/__init__.py @@ -1,9 +1,4 @@ -import logging +from mangadlp.logger import prepare_logger # prepare logger with default level INFO==20 -logging.basicConfig( - format="%(asctime)s | %(levelname)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - level=20, - handlers=[logging.StreamHandler()], -) +prepare_logger() diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index 68a6ad1..53a7983 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -1,4 +1,3 @@ -import logging import re import sys from time import sleep @@ -7,9 +6,10 @@ from typing import Any import requests import mangadlp.utils as utils +from mangadlp.logger import Logger # prepare logger -log = logging.getLogger(__name__) +log = Logger(__name__) class Mangadex: @@ -39,7 +39,7 @@ class Mangadex: # make initial request def get_manga_data(self) -> requests.Response: - log.debug(f"Getting manga data for: {self.manga_uuid}") + log.verbose(f"Getting manga data for: {self.manga_uuid}") counter = 1 while counter <= 3: try: @@ -78,7 +78,7 @@ class Mangadex: # get the title of the manga (and fix the filename) def get_manga_title(self) -> str: - log.debug(f"Getting manga title for: {self.manga_uuid}") + log.verbose(f"Getting manga title for: {self.manga_uuid}") manga_data = self.manga_data.json() try: title = manga_data["data"]["attributes"]["title"][self.language] @@ -96,7 +96,9 @@ class Mangadex: # check if chapters are available in requested language def check_chapter_lang(self) -> int: - log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}") + log.verbose( + f"Checking for chapters in specified language for: {self.manga_uuid}" + ) r = requests.get( f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}" ) @@ -116,7 +118,7 @@ class Mangadex: # get chapter data like name, uuid etc def get_chapter_data(self) -> dict: - log.debug(f"Getting chapter data for: {self.manga_uuid}") + log.verbose(f"Getting chapter data for: {self.manga_uuid}") api_sorting = "order[chapter]=asc&order[volume]=asc" # check for chapters in specified lang total_chapters = self.check_chapter_lang() @@ -174,7 +176,7 @@ class Mangadex: # get images for the chapter (mangadex@home) def get_chapter_images(self, chapter: str, wait_time: float) -> list: - log.debug(f"Getting chapter images for: {self.manga_uuid}") + log.verbose(f"Getting chapter images for: {self.manga_uuid}") athome_url = f"{self.api_base_url}/at-home/server" chapter_uuid = self.manga_chapter_data[chapter][0] @@ -220,7 +222,7 @@ class Mangadex: # create list of chapters def create_chapter_list(self) -> list: - log.debug(f"Creating chapter list for: {self.manga_uuid}") + log.verbose(f"Creating chapter list for: {self.manga_uuid}") chapter_list = [] for chapter in self.manga_chapter_data.items(): chapter_info = self.get_chapter_infos(chapter[0]) diff --git a/mangadlp/app.py b/mangadlp/app.py index 4ab364e..4559908 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -1,4 +1,3 @@ -import logging import re import shutil import sys @@ -7,12 +6,11 @@ from typing import Any import mangadlp.downloader as downloader import mangadlp.utils as utils - -# supported api's from mangadlp.api.mangadex import Mangadex +from mangadlp.logger import Logger # prepare logger -log = logging.getLogger(__name__) +log = Logger(__name__) class MangaDLP: @@ -27,7 +25,6 @@ class MangaDLP: :param forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume :param download_path: Download path. Defaults to '/downloads' :param download_wait: Time to wait for each picture to download in seconds - :param verbosity: Verbosity of the output. Uses the logging library values :return: Nothing. Just the files """ @@ -42,7 +39,6 @@ class MangaDLP: forcevol: bool = False, download_path: str = "downloads", download_wait: float = 0.5, - verbosity: int = 20, ) -> None: # init parameters self.url_uuid = url_uuid @@ -53,7 +49,6 @@ class MangaDLP: self.forcevol = forcevol self.download_path = download_path self.download_wait = download_wait - self.verbosity = verbosity # prepare everything self._prepare() @@ -110,15 +105,14 @@ class MangaDLP: # check url for match if api_mangadex.search(url_uuid) or api_mangadex2.search(url_uuid): return Mangadex - # this is only for testing multiple apis - if api_test.search(url_uuid): + elif api_test.search(url_uuid): log.critical("Not supported yet") sys.exit(1) # no supported api found log.error(f"No supported api in link/uuid found: {url_uuid}") - raise ValueError + sys.exit(1) # once called per manga def get_manga(self) -> None: @@ -129,7 +123,7 @@ class MangaDLP: print_divider = "=========================================" # show infos log.info(f"{print_divider}") - log.warning(f"Manga Name: {self.manga_title}") + log.lean(f"Manga Name: {self.manga_title}") log.info(f"Manga UUID: {self.manga_uuid}") log.info(f"Total chapters: {len(self.manga_chapter_list)}") @@ -148,7 +142,7 @@ class MangaDLP: ) # show chapters to download - log.warning(f"Chapters selected: {', '.join(chapters_to_download)}") + log.lean(f"Chapters selected: {', '.join(chapters_to_download)}") log.info(f"{print_divider}") # create manga folder @@ -173,15 +167,15 @@ class MangaDLP: # done with manga log.info(f"{print_divider}") - log.warning(f"Done with manga: {self.manga_title}") + log.lean(f"Done with manga: {self.manga_title}") # filter skipped list skipped_chapters = list(filter(None, skipped_chapters)) if len(skipped_chapters) >= 1: - log.warning(f"Skipped chapters: {', '.join(skipped_chapters)}") + log.lean(f"Skipped chapters: {', '.join(skipped_chapters)}") # filter error list error_chapters = list(filter(None, error_chapters)) if len(error_chapters) >= 1: - log.warning(f"Chapters with errors: {', '.join(error_chapters)}") + log.lean(f"Chapters with errors: {', '.join(error_chapters)}") log.info(f"{print_divider}\n") @@ -227,8 +221,7 @@ class MangaDLP: # check if chapter already exists # check for folder, if file format is an empty string if chapter_archive_path.exists(): - if self.verbosity != "lean": - log.warning(f"'{chapter_archive_path}' already exists. Skipping") + log.warning(f"'{chapter_archive_path}' already exists. Skipping") # add to skipped chapters list return ( { @@ -243,13 +236,13 @@ class MangaDLP: chapter_path.mkdir(parents=True, exist_ok=True) # verbose log - log.debug(f"Chapter UUID: {chapter_infos['uuid']}") - log.debug(f"Filename: '{chapter_archive_path.name}'") - log.debug(f"File path: '{chapter_archive_path}'") - log.debug(f"Image URLS:\n{chapter_image_urls}") + log.verbose(f"Chapter UUID: {chapter_infos['uuid']}") + log.verbose(f"Filename: '{chapter_archive_path.name}'") + log.verbose(f"File path: '{chapter_archive_path}'") + log.verbose(f"Image URLS:\n{chapter_image_urls}") # log - log.warning(f"Downloading: '{chapter_filename}'") + log.lean(f"Downloading: '{chapter_filename}'") # download images try: @@ -273,12 +266,12 @@ class MangaDLP: else: # Done with chapter - log.warning(f"Successfully downloaded: '{chapter_filename}'") + log.lean(f"Successfully downloaded: '{chapter_filename}'") return {"chapter_path": chapter_path} # create an archive of the chapter if needed def archive_chapter(self, chapter_path: Path) -> dict: - log.warning(f"Creating archive '{chapter_path}{self.file_format}'") + log.lean(f"Creating archive '{chapter_path}{self.file_format}'") try: # check if image folder is existing if not chapter_path.exists(): diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index 14fa08b..d633212 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -8,9 +8,10 @@ from typing import Union import requests import mangadlp.utils as utils +from mangadlp.logger import Logger # prepare logger -log = logging.getLogger(__name__) +log = Logger(__name__) # download images def download_chapter( @@ -27,7 +28,7 @@ def download_chapter( # show progress bar for default log level if logging.root.level == logging.INFO: utils.progress_bar(image_num, total_img) - log.debug(f"Downloading image {image_num}/{total_img}") + log.verbose(f"Downloading image {image_num}/{total_img}") counter = 1 while counter <= 3: diff --git a/mangadlp/input.py b/mangadlp/input.py index aa0128a..9fd4cde 100644 --- a/mangadlp/input.py +++ b/mangadlp/input.py @@ -1,13 +1,13 @@ import argparse -import logging import sys from pathlib import Path import mangadlp.app as app import mangadlp.logger as logger +from mangadlp.logger import Logger # prepare logger -log = logging.getLogger(__name__) +log = Logger(__name__) MDLP_VERSION = "2.1.10" @@ -34,7 +34,7 @@ def check_args(args): # read in the list of links from a file def readin_list(readlist: str) -> list: list_file = Path(readlist) - log.debug(f"Reading in list '{str(list_file)}'") + log.verbose(f"Reading in list '{str(list_file)}'") try: url_str = list_file.read_text() url_list = url_str.splitlines() @@ -43,7 +43,7 @@ def readin_list(readlist: str) -> list: # filter empty lines and remove them filtered_list = list(filter(len, url_list)) - log.debug(f"Mangas from list: {filtered_list}") + log.verbose(f"Mangas from list: {filtered_list}") return filtered_list @@ -59,7 +59,6 @@ def call_app(args): args.forcevol, args.path, args.wait, - args.verbosity, ) mdlp.get_manga() @@ -196,7 +195,16 @@ def get_args(): required=False, help="Lean logging. Minimal log output. Defaults to false", action="store_const", - const=30, + const=25, + default=20, + ) + verbosity.add_argument( + "--verbose", + dest="verbosity", + required=False, + help="Verbose logging. More log output. Defaults to false", + action="store_const", + const=15, default=20, ) verbosity.add_argument( diff --git a/mangadlp/logger.py b/mangadlp/logger.py index 1c175f2..769f3c5 100644 --- a/mangadlp/logger.py +++ b/mangadlp/logger.py @@ -1,6 +1,18 @@ import logging +# prepare custom levels and default config of logger +def prepare_logger(): + logging.basicConfig( + format="%(asctime)s | %(levelname)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=20, + handlers=[logging.StreamHandler()], + ) + logging.addLevelName(level=15, levelName="VERBOSE") + logging.addLevelName(level=25, levelName="LEAN") + + # set log message format def format_logger(verbosity: int): logging.getLogger().setLevel(verbosity) @@ -18,3 +30,33 @@ def format_logger(verbosity: int): datefmt="%Y-%m-%d %H:%M:%S", force=True, ) + + +class Logger: + def __init__(self, name: str): + self.name = name + # create logger + self.log = logging.getLogger(self.name) + + # custom log levels + def verbose(self, message: str): + self.log.log(level=15, msg=message) + + def lean(self, message: str): + self.log.log(level=25, msg=message) + + # default log levels + def critical(self, message: str): + self.log.critical(msg=message) + + def error(self, message: str): + self.log.error(msg=message) + + def warning(self, message: str): + self.log.warning(msg=message) + + def info(self, message: str): + self.log.info(msg=message) + + def debug(self, message: str): + self.log.debug(msg=message) diff --git a/mangadlp/utils.py b/mangadlp/utils.py index 236231a..05b8125 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -1,12 +1,13 @@ -import logging import re from datetime import datetime from pathlib import Path from typing import Any from zipfile import ZipFile +from mangadlp.logger import Logger + # prepare logger -log = logging.getLogger(__name__) +log = Logger(__name__) # create an archive of the chapter images def make_archive(chapter_path: Path, file_format: str) -> None: diff --git a/tests/test_01_app.py b/tests/test_01_app.py index a68c9bc..0115d5f 100644 --- a/tests/test_01_app.py +++ b/tests/test_01_app.py @@ -13,6 +13,7 @@ def test_check_api_mangadex(): def test_check_api_none(): url = "https://abc.defghjk/title/abc/def" - with pytest.raises(ValueError) as e: + with pytest.raises(SystemExit) as e: app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2) - assert e.type == ValueError + assert e.type == SystemExit + assert e.value.code == 1 diff --git a/tests/test_02_utils.py b/tests/test_02_utils.py index 4c12d23..ee8ed9c 100644 --- a/tests/test_02_utils.py +++ b/tests/test_02_utils.py @@ -57,7 +57,6 @@ def test_chapter_list_full(): forcevol=True, download_path="tests", download_wait=2, - verbosity=10, ) chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list) assert chap_list == [ diff --git a/tests/test_21_full.py b/tests/test_21_full.py index 2ba743f..f7aacf6 100644 --- a/tests/test_21_full.py +++ b/tests/test_21_full.py @@ -33,7 +33,6 @@ def test_full_api_mangadex(wait_20s): forcevol=False, download_path="tests", download_wait=2, - verbosity=10, ) mdlp.get_manga()