olofvndrhr/manga-dlp
1
0
Fork 0
Code Issues 1 Pull requests 10 Projects Releases 31 Packages Wiki Activity Actions

first adjustments to the logger

Browse source
This commit is contained in:
Ivan Schaller 2022-07-15 12:49:49 +02:00
parent 8972556415
commit 66e916a580
7 changed files with 84 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
mangadlp/__init__.py
View file

@ -1,7 +1,5 @@
import logging
from mangadlp.logger import logger_lean, logger_verbose
# prepare logger with default level INFO==20
logging.basicConfig(
format="%(asctime)s | %(levelname)s: %(message)s",
@ -9,14 +7,3 @@ logging.basicConfig(
level=20,
handlers=[logging.StreamHandler()],
)
# create custom log levels
logging.addLevelName(15, "VERBOSE")
logging.VERBOSE = 15 # type: ignore
logging.verbose = logger_verbose # type: ignore
logging.Logger.verbose = logger_verbose # type: ignore
logging.addLevelName(25, "LEAN")
logging.VERBOSE = 25 # type: ignore
logging.lean = logger_lean # type: ignore
logging.Logger.lean = logger_lean # type: ignore

41
mangadlp/api/mangadex.py
View file

@ -8,6 +8,9 @@ import requests
import mangadlp.utils as utils
# prepare logger
log = logging.getLogger(__name__)
class Mangadex:
@ -36,7 +39,7 @@ class Mangadex:
# make initial request
def get_manga_data(self) -> requests.Response:
logging.verbose(f"Getting manga data for: {self.manga_uuid}") # type: ignore
log.debug(f"Getting manga data for: {self.manga_uuid}")
counter = 1
while counter <= 3:
try:
@ -45,17 +48,17 @@ class Mangadex:
)
except:
if counter >= 3:
logging.error("Maybe the MangaDex API is down?")
log.error("Maybe the MangaDex API is down?")
sys.exit(1)
else:
logging.error("Mangadex API not reachable. Retrying")
log.error("Mangadex API not reachable. Retrying")
sleep(2)
counter += 1
else:
break
# check if manga exists
if manga_data.json()["result"] != "ok":
logging.error("Manga not found")
log.error("Manga not found")
sys.exit(1)
return manga_data
@ -68,14 +71,14 @@ class Mangadex:
)
# check for new mangadex id
if not uuid_regex.search(self.url_uuid):
logging.error("No valid UUID found")
log.error("No valid UUID found")
sys.exit(1)
manga_uuid = uuid_regex.search(self.url_uuid)[0]
return manga_uuid
# get the title of the manga (and fix the filename)
def get_manga_title(self) -> str:
logging.verbose(f"Getting manga title for: {self.manga_uuid}") # type: ignore
log.debug(f"Getting manga title for: {self.manga_uuid}")
manga_data = self.manga_data.json()
try:
title = manga_data["data"]["attributes"]["title"][self.language]
@ -87,35 +90,33 @@ class Mangadex:
alt_titles.update(title)
title = alt_titles[self.language]
except: # no title on requested language found
logging.error("Chapter in requested language not found.")
log.error("Chapter in requested language not found.")
sys.exit(1)
return utils.fix_name(title)
# check if chapters are available in requested language
def check_chapter_lang(self) -> int:
logging.verbose( # type: ignore
f"Checking for chapters in specified language for: {self.manga_uuid}"
)
log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}")
r = requests.get(
f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}"
)
try:
total_chapters = r.json()["total"]
except:
logging.error(
log.error(
"Error retrieving the chapters list. Did you specify a valid language code?"
)
return 0
else:
if total_chapters == 0:
logging.error("No chapters available to download!")
log.error("No chapters available to download!")
return 0
return total_chapters
# get chapter data like name, uuid etc
def get_chapter_data(self) -> dict:
logging.verbose(f"Getting chapter data for: {self.manga_uuid}") # type: ignore
log.debug(f"Getting chapter data for: {self.manga_uuid}")
api_sorting = "order[chapter]=asc&order[volume]=asc"
# check for chapters in specified lang
total_chapters = self.check_chapter_lang()
@ -173,7 +174,7 @@ class Mangadex:
# get images for the chapter (mangadex@home)
def get_chapter_images(self, chapter: str, wait_time: float) -> list:
logging.verbose(f"Getting chapter images for: {self.manga_uuid}") # type: ignore
log.debug(f"Getting chapter images for: {self.manga_uuid}")
athome_url = f"{self.api_base_url}/at-home/server"
chapter_uuid = self.manga_chapter_data[chapter][0]
@ -185,11 +186,11 @@ class Mangadex:
r = requests.get(f"{athome_url}/{chapter_uuid}")
api_data = r.json()
if api_data["result"] != "ok":
logging.error(f"No chapter with the id {chapter_uuid} found")
log.error(f"No chapter with the id {chapter_uuid} found")
api_error = True
raise IndexError
elif api_data["chapter"]["data"] is None:
logging.error(f"No chapter data found for chapter {chapter_uuid}")
log.error(f"No chapter data found for chapter {chapter_uuid}")
api_error = True
raise IndexError
else:
@ -198,7 +199,7 @@ class Mangadex:
except:
if counter >= 3:
api_error = True
logging.error(f"Retrying in a few seconds")
log.error(f"Retrying in a few seconds")
counter += 1
sleep(wait_time + 2)
# check if result is ok
@ -219,7 +220,7 @@ class Mangadex:
# create list of chapters
def create_chapter_list(self) -> list:
logging.verbose(f"Creating chapter list for: {self.manga_uuid}") # type: ignore
log.debug(f"Creating chapter list for: {self.manga_uuid}")
chapter_list = []
for chapter in self.manga_chapter_data.items():
chapter_info = self.get_chapter_infos(chapter[0])
@ -234,9 +235,7 @@ class Mangadex:
# create easy to access chapter infos
def get_chapter_infos(self, chapter: str) -> dict:
logging.debug(
f"Getting chapter infos for: {self.manga_chapter_data[chapter][0]}"
)
log.debug(f"Getting chapter infos for: {self.manga_chapter_data[chapter][0]}")
chapter_uuid = self.manga_chapter_data[chapter][0]
chapter_vol = self.manga_chapter_data[chapter][1]
chapter_num = self.manga_chapter_data[chapter][2]

73
mangadlp/app.py
View file

@ -11,6 +11,9 @@ import mangadlp.utils as utils
# supported api's
from mangadlp.api.mangadex import Mangadex
# prepare logger
log = logging.getLogger(__name__)
class MangaDLP:
"""Download Mangas from supported sites.
@ -74,25 +77,25 @@ class MangaDLP:
# prechecks userinput/options
# no url and no readin list given
if not self.url_uuid:
logging.error(
log.error(
'You need to specify a manga url/uuid with "-u" or a list with "--read"'
)
sys.exit(1)
# checks if --list is not used
if not self.list_chapters:
if self.chapters is None:
if not self.chapters:
# no chapters to download were given
logging.error(
log.error(
'You need to specify one or more chapters to download. To see all chapters use "--list"'
)
sys.exit(1)
# if forcevol is used, but didn't specify a volume in the chapters selected
if self.forcevol and ":" not in self.chapters:
logging.error("You need to specify the volume if you use --forcevol")
log.error("You need to specify the volume if you use --forcevol")
sys.exit(1)
# if forcevol is not used, but a volume is specified
if not self.forcevol and ":" in self.chapters:
logging.error("Don't specify the volume without --forcevol")
log.error("Don't specify the volume without --forcevol")
sys.exit(1)
# check the api which needs to be used
@ -110,11 +113,11 @@ class MangaDLP:
# this is only for testing multiple apis
if api_test.search(url_uuid):
logging.critical("Not supported yet")
log.critical("Not supported yet")
sys.exit(1)
# no supported api found
logging.error(f"No supported api in link/uuid found: {url_uuid}")
log.error(f"No supported api in link/uuid found: {url_uuid}")
raise ValueError
# once called per manga
@ -125,15 +128,15 @@ class MangaDLP:
print_divider = "========================================="
# show infos
logging.info(f"{print_divider}")
logging.lean(f"Manga Name: {self.manga_title}") # type: ignore
logging.info(f"Manga UUID: {self.manga_uuid}")
logging.info(f"Total chapters: {len(self.manga_chapter_list)}")
log.info(f"{print_divider}")
log.warning(f"Manga Name: {self.manga_title}")
log.info(f"Manga UUID: {self.manga_uuid}")
log.info(f"Total chapters: {len(self.manga_chapter_list)}")
# list chapters if list_chapters is true
if self.list_chapters:
logging.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
logging.info(f"{print_divider}\n")
log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}")
log.info(f"{print_divider}\n")
return None
# check chapters to download if not all
@ -145,8 +148,8 @@ class MangaDLP:
)
# show chapters to download
logging.lean(f"Chapters selected: {', '.join(chapters_to_download)}") # type: ignore
logging.info(f"{print_divider}")
log.warning(f"Chapters selected: {', '.join(chapters_to_download)}")
log.info(f"{print_divider}")
# create manga folder
self.manga_path.mkdir(parents=True, exist_ok=True)
@ -166,21 +169,21 @@ class MangaDLP:
# chapter was not skipped
except KeyError:
# done with chapter
logging.info("Done with chapter\n")
log.info(f"Done with chapter '{chapter}'\n")
# done with manga
logging.info(f"{print_divider}")
logging.lean(f"Done with manga: {self.manga_title}") # type: ignore
log.info(f"{print_divider}")
log.warning(f"Done with manga: {self.manga_title}")
# filter skipped list
skipped_chapters = list(filter(None, skipped_chapters))
if len(skipped_chapters) >= 1:
logging.lean(f"Skipped chapters: {', '.join(skipped_chapters)}") # type: ignore
log.warning(f"Skipped chapters: {', '.join(skipped_chapters)}")
# filter error list
error_chapters = list(filter(None, error_chapters))
if len(error_chapters) >= 1:
logging.lean(f"Chapters with errors: {', '.join(error_chapters)}") # type: ignore
log.warning(f"Chapters with errors: {', '.join(error_chapters)}")
logging.info(f"{print_divider}\n")
log.info(f"{print_divider}\n")
# once called per chapter
def get_chapter(self, chapter: str) -> dict:
@ -193,12 +196,12 @@ class MangaDLP:
chapter, self.download_wait
)
except KeyboardInterrupt:
logging.critical("Stopping")
log.critical("Stopping")
sys.exit(1)
# check if the image urls are empty. if yes skip this chapter (for mass downloads)
if not chapter_image_urls:
logging.error(
log.error(
f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}"
)
# add to skipped chapters list
@ -225,7 +228,7 @@ class MangaDLP:
# check for folder, if file format is an empty string
if chapter_archive_path.exists():
if self.verbosity != "lean":
logging.warning(f"'{chapter_archive_path}' already exists. Skipping")
log.warning(f"'{chapter_archive_path}' already exists. Skipping")
# add to skipped chapters list
return (
{
@ -240,13 +243,13 @@ class MangaDLP:
chapter_path.mkdir(parents=True, exist_ok=True)
# verbose log
logging.verbose(f"Chapter UUID: {chapter_infos['uuid']}") # type: ignore
logging.verbose(f"Filename: '{chapter_archive_path.name}'") # type: ignore
logging.verbose(f"File path: '{chapter_archive_path}'") # type: ignore
logging.verbose(f"Image URLS:\n{chapter_image_urls}") # type: ignore
log.debug(f"Chapter UUID: {chapter_infos['uuid']}")
log.debug(f"Filename: '{chapter_archive_path.name}'")
log.debug(f"File path: '{chapter_archive_path}'")
log.debug(f"Image URLS:\n{chapter_image_urls}")
# log
logging.lean(f"Downloading: '{chapter_filename}'") # type: ignore
log.warning(f"Downloading: '{chapter_filename}'")
# download images
try:
@ -254,10 +257,10 @@ class MangaDLP:
chapter_image_urls, chapter_path, self.download_wait
)
except KeyboardInterrupt:
logging.critical("Stopping")
log.critical("Stopping")
sys.exit(1)
except:
logging.error(f"Cant download: '{chapter_filename}'. Skipping")
log.error(f"Cant download: '{chapter_filename}'. Skipping")
# add to skipped chapters list
return (
{
@ -270,23 +273,23 @@ class MangaDLP:
else:
# Done with chapter
logging.lean(f"INFO: Successfully downloaded: '{chapter_filename}'") # type: ignore
log.warning(f"Successfully downloaded: '{chapter_filename}'")
return {"chapter_path": chapter_path}
# create an archive of the chapter if needed
def archive_chapter(self, chapter_path: Path) -> dict:
logging.lean(f"INFO: Creating '{self.file_format}' archive") # type: ignore
log.warning(f"Creating archive '{chapter_path}{self.file_format}'")
try:
# check if image folder is existing
if not chapter_path.exists():
logging.error(f"Image folder: {chapter_path} does not exist")
log.error(f"Image folder: {chapter_path} does not exist")
raise IOError
if self.file_format == ".pdf":
utils.make_pdf(chapter_path)
else:
utils.make_archive(chapter_path, self.file_format)
except:
logging.error(f"Archive error. Skipping chapter")
log.error(f"Archive error. Skipping chapter")
# add to skipped chapters list
return {
"error": chapter_path,

12
mangadlp/downloader.py
View file

@ -9,6 +9,8 @@ import requests
import mangadlp.utils as utils
# prepare logger
log = logging.getLogger(__name__)
# download images
def download_chapter(
@ -25,21 +27,21 @@ def download_chapter(
# show progress bar for default log level
if logging.root.level == logging.INFO:
utils.progress_bar(image_num, total_img)
logging.verbose(f"Downloading image {image_num}/{total_img}") # type: ignore
log.debug(f"Downloading image {image_num}/{total_img}")
counter = 1
while counter <= 3:
try:
r = requests.get(image, stream=True)
if r.status_code != 200:
logging.error(f"Request for image {image} failed, retrying")
log.error(f"Request for image {image} failed, retrying")
raise ConnectionError
except KeyboardInterrupt:
logging.critical("Stopping")
log.critical("Stopping")
sys.exit(1)
except:
if counter >= 3:
logging.error("Maybe the MangaDex Servers are down?")
log.error("Maybe the MangaDex Servers are down?")
raise ConnectionError
sleep(download_wait)
counter += 1
@ -52,7 +54,7 @@ def download_chapter(
r.raw.decode_content = True
shutil.copyfileobj(r.raw, file)
except:
logging.error("Can't write file")
log.error("Can't write file")
raise IOError
image_num += 1

27
mangadlp/input.py
View file

@ -1,14 +1,20 @@
import argparse
import logging
import sys
from pathlib import Path
import mangadlp.app as app
import mangadlp.logger as logger
# prepare logger
log = logging.getLogger(__name__)
MDLP_VERSION = "2.1.10"
def check_args(args):
# set logger formatting
logger.format_logger(args.verbosity)
# check if --version was used
if args.version:
print(f"manga-dlp version: {MDLP_VERSION}")
@ -28,18 +34,21 @@ def check_args(args):
# read in the list of links from a file
def readin_list(readlist: str) -> list:
list_file = Path(readlist)
log.debug(f"Reading in list '{str(list_file)}'")
try:
url_str = list_file.read_text()
url_list = url_str.splitlines()
except:
raise IOError
return url_list
# filter empty lines and remove them
filtered_list = list(filter(len, url_list))
log.debug(f"Mangas from list: {filtered_list}")
return filtered_list
def call_app(args):
# set logger formatting
logger.format_logger(args.verbosity)
# call main function with all input arguments
mdlp = app.MangaDLP(
args.url_uuid,
@ -90,6 +99,7 @@ def get_input():
# start script again with the arguments
sys.argv.extend(args)
log.info(f"Args: {sys.argv}")
get_args()
@ -186,16 +196,7 @@ def get_args():
required=False,
help="Lean logging. Minimal log output. Defaults to false",
action="store_const",
const=25,
default=20,
)
verbosity.add_argument(
"--verbose",
dest="verbosity",
required=False,
help="Verbose logging. More log output. Defaults to false",
action="store_const",
const=15,
const=30,
default=20,
)
verbosity.add_argument(

14
mangadlp/logger.py
View file

@ -14,19 +14,7 @@ def format_logger(verbosity: int):
)
else:
logging.basicConfig(
format="%(asctime)s | %(levelname)s: %(message)s",
format="%(asctime)s | [%(name)s] %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
# create verbose logger with level 15
def logger_verbose(msg, *args, **kwargs):
if logging.getLogger().isEnabledFor(15):
logging.log(15, msg)
# create lean logger with level 25
def logger_lean(msg, *args, **kwargs):
if logging.getLogger().isEnabledFor(25):
logging.log(25, msg)

6
mangadlp/utils.py
View file

@ -5,6 +5,8 @@ from pathlib import Path
from typing import Any
from zipfile import ZipFile
# prepare logger
log = logging.getLogger(__name__)
# create an archive of the chapter images
def make_archive(chapter_path: Path, file_format: str) -> None:
@ -24,7 +26,7 @@ def make_pdf(chapter_path: Path) -> None:
try:
import img2pdf
except:
logging.error("Cant import img2pdf. Please install it first")
log.error("Cant import img2pdf. Please install it first")
raise ImportError
pdf_path = Path(f"{chapter_path}.pdf")
@ -34,7 +36,7 @@ def make_pdf(chapter_path: Path) -> None:
try:
pdf_path.write_bytes(img2pdf.convert(images))
except:
logging.error("Can't create '.pdf' archive")
log.error("Can't create '.pdf' archive")
raise IOError