olofvndrhr/manga-dlp
1
0
Fork 0
Code Issues 1 Pull requests 10 Projects Releases 31 Packages Wiki Activity Actions

logging improvements
All checks were successful
ci/woodpecker/push/tests Pipeline was successful
Details

Browse source
This commit is contained in:
Ivan Schaller 2022-07-15 14:04:22 +02:00
parent 66e916a580
commit 820c891fd7
12 changed files with 117 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
CHANGELOG.md
View file

@ -9,6 +9,23 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- Add support for more sites
## [2.1.11] - 2022-07-
### Fixed
- The `--read` option now filters empty lines, so it will not generate an error anymore
- An error which was caused by the interactive input method when you did not specify a chapter or to list them
### Added
- `autoflake` test in `justfile`
- Some more things which get logged
### Changed
- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other
improvements
## [2.1.10] - 2022-07-14
### Fixed

6
justfile
View file

@ -82,6 +82,10 @@ test_mypy:
test_pytest:
@python3 -m tox -e basic
test_autoflake:
@python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/
@python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/
test_tox:
@python3 -m tox
@ -111,6 +115,7 @@ lint:
just test_black
just test_isort
just test_mypy
just test_autoflake
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"
tests:
@ -120,6 +125,7 @@ tests:
just test_black
just test_isort
just test_mypy
just test_autoflake
just test_pytest
@echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n"

9
mangadlp/__init__.py
View file

@ -1,9 +1,4 @@
import logging
from mangadlp.logger import prepare_logger
# prepare logger with default level INFO==20
logging.basicConfig(
format="%(asctime)s | %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=20,
handlers=[logging.StreamHandler()],
)
prepare_logger()

18
mangadlp/api/mangadex.py
View file

@ -1,4 +1,3 @@
import logging
import re
import sys
from time import sleep
@ -7,9 +6,10 @@ from typing import Any
import requests
import mangadlp.utils as utils
from mangadlp.logger import Logger
# prepare logger
log = logging.getLogger(__name__)
log = Logger(__name__)
class Mangadex:
@ -39,7 +39,7 @@ class Mangadex:
# make initial request
def get_manga_data(self) -> requests.Response:
log.debug(f"Getting manga data for: {self.manga_uuid}")
log.verbose(f"Getting manga data for: {self.manga_uuid}")
counter = 1
while counter <= 3:
try:
@ -78,7 +78,7 @@ class Mangadex:
# get the title of the manga (and fix the filename)
def get_manga_title(self) -> str:
log.debug(f"Getting manga title for: {self.manga_uuid}")
log.verbose(f"Getting manga title for: {self.manga_uuid}")
manga_data = self.manga_data.json()
try:
title = manga_data["data"]["attributes"]["title"][self.language]
@ -96,7 +96,9 @@ class Mangadex:
# check if chapters are available in requested language
def check_chapter_lang(self) -> int:
log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}")
log.verbose(
f"Checking for chapters in specified language for: {self.manga_uuid}"
)
r = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}"
)
@ -116,7 +118,7 @@ class Mangadex:
# get chapter data like name, uuid etc
def get_chapter_data(self) -> dict:
log.debug(f"Getting chapter data for: {self.manga_uuid}")
log.verbose(f"Getting chapter data for: {self.manga_uuid}")
api_sorting = "order[chapter]=asc&order[volume]=asc"
# check for chapters in specified lang
total_chapters = self.check_chapter_lang()
@ -174,7 +176,7 @@ class Mangadex:
# get images for the chapter (mangadex@home)
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
log.debug(f"Getting chapter images for: {self.manga_uuid}")
log.verbose(f"Getting chapter images for: {self.manga_uuid}")
athome_url = f"{self.api_base_url}/at-home/server"
chapter_uuid = self.manga_chapter_data[chapter][0]
@ -220,7 +222,7 @@ class Mangadex:
# create list of chapters
def create_chapter_list(self) -> list:
log.debug(f"Creating chapter list for: {self.manga_uuid}")
log.verbose(f"Creating chapter list for: {self.manga_uuid}")
chapter_list = []
for chapter in self.manga_chapter_data.items():
chapter_info = self.get_chapter_infos(chapter[0])

39
mangadlp/app.py
View file

@ -1,4 +1,3 @@
import logging
import re
import shutil
import sys
@ -7,12 +6,11 @@ from typing import Any
import mangadlp.downloader as downloader
import mangadlp.utils as utils
# supported api's
from mangadlp.api.mangadex import Mangadex
from mangadlp.logger import Logger
# prepare logger
log = logging.getLogger(__name__)
log = Logger(__name__)
class MangaDLP:
@ -27,7 +25,6 @@ class MangaDLP:
:param forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume
:param download_path: Download path. Defaults to '<script_dir>/downloads'
:param download_wait: Time to wait for each picture to download in seconds
:param verbosity: Verbosity of the output. Uses the logging library values
:return: Nothing. Just the files
"""
@ -42,7 +39,6 @@ class MangaDLP:
forcevol: bool = False,
download_path: str = "downloads",
download_wait: float = 0.5,
verbosity: int = 20,
) -> None:
# init parameters
self.url_uuid = url_uuid
@ -53,7 +49,6 @@ class MangaDLP:
self.forcevol = forcevol
self.download_path = download_path
self.download_wait = download_wait
self.verbosity = verbosity
# prepare everything
self._prepare()
@ -110,15 +105,14 @@ class MangaDLP:
# check url for match
if api_mangadex.search(url_uuid) or api_mangadex2.search(url_uuid):
return Mangadex
# this is only for testing multiple apis
if api_test.search(url_uuid):
elif api_test.search(url_uuid):
log.critical("Not supported yet")
sys.exit(1)
# no supported api found
log.error(f"No supported api in link/uuid found: {url_uuid}")
raise ValueError
sys.exit(1)
# once called per manga
def get_manga(self) -> None:
@ -129,7 +123,7 @@ class MangaDLP:
print_divider = "========================================="
# show infos
log.info(f"{print_divider}")
log.warning(f"Manga Name: {self.manga_title}")
log.lean(f"Manga Name: {self.manga_title}")
log.info(f"Manga UUID: {self.manga_uuid}")
log.info(f"Total chapters: {len(self.manga_chapter_list)}")
@ -148,7 +142,7 @@ class MangaDLP:
)
# show chapters to download
log.warning(f"Chapters selected: {', '.join(chapters_to_download)}")
log.lean(f"Chapters selected: {', '.join(chapters_to_download)}")
log.info(f"{print_divider}")
# create manga folder
@ -173,15 +167,15 @@ class MangaDLP:
# done with manga
log.info(f"{print_divider}")
log.warning(f"Done with manga: {self.manga_title}")
log.lean(f"Done with manga: {self.manga_title}")
# filter skipped list
skipped_chapters = list(filter(None, skipped_chapters))
if len(skipped_chapters) >= 1:
log.warning(f"Skipped chapters: {', '.join(skipped_chapters)}")
log.lean(f"Skipped chapters: {', '.join(skipped_chapters)}")
# filter error list
error_chapters = list(filter(None, error_chapters))
if len(error_chapters) >= 1:
log.warning(f"Chapters with errors: {', '.join(error_chapters)}")
log.lean(f"Chapters with errors: {', '.join(error_chapters)}")
log.info(f"{print_divider}\n")
@ -227,7 +221,6 @@ class MangaDLP:
# check if chapter already exists
# check for folder, if file format is an empty string
if chapter_archive_path.exists():
if self.verbosity != "lean":
log.warning(f"'{chapter_archive_path}' already exists. Skipping")
# add to skipped chapters list
return (
@ -243,13 +236,13 @@ class MangaDLP:
chapter_path.mkdir(parents=True, exist_ok=True)
# verbose log
log.debug(f"Chapter UUID: {chapter_infos['uuid']}")
log.debug(f"Filename: '{chapter_archive_path.name}'")
log.debug(f"File path: '{chapter_archive_path}'")
log.debug(f"Image URLS:\n{chapter_image_urls}")
log.verbose(f"Chapter UUID: {chapter_infos['uuid']}")
log.verbose(f"Filename: '{chapter_archive_path.name}'")
log.verbose(f"File path: '{chapter_archive_path}'")
log.verbose(f"Image URLS:\n{chapter_image_urls}")
# log
log.warning(f"Downloading: '{chapter_filename}'")
log.lean(f"Downloading: '{chapter_filename}'")
# download images
try:
@ -273,12 +266,12 @@ class MangaDLP:
else:
# Done with chapter
log.warning(f"Successfully downloaded: '{chapter_filename}'")
log.lean(f"Successfully downloaded: '{chapter_filename}'")
return {"chapter_path": chapter_path}
# create an archive of the chapter if needed
def archive_chapter(self, chapter_path: Path) -> dict:
log.warning(f"Creating archive '{chapter_path}{self.file_format}'")
log.lean(f"Creating archive '{chapter_path}{self.file_format}'")
try:
# check if image folder is existing
if not chapter_path.exists():

5
mangadlp/downloader.py
View file

@ -8,9 +8,10 @@ from typing import Union
import requests
import mangadlp.utils as utils
from mangadlp.logger import Logger
# prepare logger
log = logging.getLogger(__name__)
log = Logger(__name__)
# download images
def download_chapter(
@ -27,7 +28,7 @@ def download_chapter(
# show progress bar for default log level
if logging.root.level == logging.INFO:
utils.progress_bar(image_num, total_img)
log.debug(f"Downloading image {image_num}/{total_img}")
log.verbose(f"Downloading image {image_num}/{total_img}")
counter = 1
while counter <= 3:

20
mangadlp/input.py
View file

@ -1,13 +1,13 @@
import argparse
import logging
import sys
from pathlib import Path
import mangadlp.app as app
import mangadlp.logger as logger
from mangadlp.logger import Logger
# prepare logger
log = logging.getLogger(__name__)
log = Logger(__name__)
MDLP_VERSION = "2.1.10"
@ -34,7 +34,7 @@ def check_args(args):
# read in the list of links from a file
def readin_list(readlist: str) -> list:
list_file = Path(readlist)
log.debug(f"Reading in list '{str(list_file)}'")
log.verbose(f"Reading in list '{str(list_file)}'")
try:
url_str = list_file.read_text()
url_list = url_str.splitlines()
@ -43,7 +43,7 @@ def readin_list(readlist: str) -> list:
# filter empty lines and remove them
filtered_list = list(filter(len, url_list))
log.debug(f"Mangas from list: {filtered_list}")
log.verbose(f"Mangas from list: {filtered_list}")
return filtered_list
@ -59,7 +59,6 @@ def call_app(args):
args.forcevol,
args.path,
args.wait,
args.verbosity,
)
mdlp.get_manga()
@ -196,7 +195,16 @@ def get_args():
required=False,
help="Lean logging. Minimal log output. Defaults to false",
action="store_const",
const=30,
const=25,
default=20,
)
verbosity.add_argument(
"--verbose",
dest="verbosity",
required=False,
help="Verbose logging. More log output. Defaults to false",
action="store_const",
const=15,
default=20,
)
verbosity.add_argument(

42
mangadlp/logger.py
View file

@ -1,6 +1,18 @@
import logging
# prepare custom levels and default config of logger
def prepare_logger():
logging.basicConfig(
format="%(asctime)s | %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=20,
handlers=[logging.StreamHandler()],
)
logging.addLevelName(level=15, levelName="VERBOSE")
logging.addLevelName(level=25, levelName="LEAN")
# set log message format
def format_logger(verbosity: int):
logging.getLogger().setLevel(verbosity)
@ -18,3 +30,33 @@ def format_logger(verbosity: int):
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
class Logger:
def __init__(self, name: str):
self.name = name
# create logger
self.log = logging.getLogger(self.name)
# custom log levels
def verbose(self, message: str):
self.log.log(level=15, msg=message)
def lean(self, message: str):
self.log.log(level=25, msg=message)
# default log levels
def critical(self, message: str):
self.log.critical(msg=message)
def error(self, message: str):
self.log.error(msg=message)
def warning(self, message: str):
self.log.warning(msg=message)
def info(self, message: str):
self.log.info(msg=message)
def debug(self, message: str):
self.log.debug(msg=message)

5
mangadlp/utils.py
View file

@ -1,12 +1,13 @@
import logging
import re
from datetime import datetime
from pathlib import Path
from typing import Any
from zipfile import ZipFile
from mangadlp.logger import Logger
# prepare logger
log = logging.getLogger(__name__)
log = Logger(__name__)
# create an archive of the chapter images
def make_archive(chapter_path: Path, file_format: str) -> None:

5
tests/test_01_app.py
View file

@ -13,6 +13,7 @@ def test_check_api_mangadex():
def test_check_api_none():
url = "https://abc.defghjk/title/abc/def"
with pytest.raises(ValueError) as e:
with pytest.raises(SystemExit) as e:
app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2)
assert e.type == ValueError
assert e.type == SystemExit
assert e.value.code == 1

1
tests/test_02_utils.py
View file

@ -57,7 +57,6 @@ def test_chapter_list_full():
forcevol=True,
download_path="tests",
download_wait=2,
verbosity=10,
)
chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list)
assert chap_list == [

1
tests/test_21_full.py
View file

@ -33,7 +33,6 @@ def test_full_api_mangadex(wait_20s):
forcevol=False,
download_path="tests",
download_wait=2,
verbosity=10,
)
mdlp.get_manga()