diff --git a/.tool-versions b/.tool-versions index 06d8fd0..e895ebd 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,4 +1,4 @@ -python 3.9.13 3.10.5 3.8.13 3.7.13 3.6.15 +python 3.9.13 3.10.5 3.8.13 shfmt 3.5.1 shellcheck 0.8.0 just 1.2.0 diff --git a/README.md b/README.md index d1d1cf8..663f170 100644 --- a/README.md +++ b/README.md @@ -89,30 +89,34 @@ See the docker [README](https://manga-dlp.ivn.sh/docker/) ## Options ```txt -usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--lean | --verbose | --debug] [--hook-manga-pre HOOK_MANGA_PRE] -[--hook-manga-post HOOK_MANGA_POST] [--hook-chapter-pre HOOK_CHAPTER_PRE] [--hook-chapter-post HOOK_CHAPTER_POST] +Usage: manga-dlp.py [OPTIONS] Script to download mangas from various sites -optional arguments: --h, --help show this help message and exit --u URL_UUID, --url URL_UUID, --uuid URL_UUID URL or UUID of the manga ---read READ Path of file with manga links to download. One per line --v, --version Show version of manga-dlp and exit --c CHAPTERS, --chapters CHAPTERS Chapters to download --p PATH, --path PATH Download path. Defaults to "/downloads" --l LANG, --language LANG Manga language. Defaults to "en" --> english ---list List all available chapters. Defaults to false ---format FORMAT Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz' ---forcevol Force naming of volumes. For mangas where chapters reset each volume ---wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5 ---lean Lean logging. Minimal log output. Defaults to false ---verbose Verbose logging. More log output. Defaults to false ---debug Debug logging. Most log output. Defaults to false ---hook-manga-pre HOOK_MANGA_PRE Commands to execute before the manga download starts ---hook-manga-post HOOK_MANGA_POST Commands to execute after the manga download finished ---hook-chapter-pre HOOK_CHAPTER_PRE Commands to execute before the chapter download starts ---hook-chapter-post HOOK_CHAPTER_POST Commands to execute after the chapter download finished +Options: +--help Show this message and exit. +--version Show the version and exit. + +source: [mutually_exclusive, required] +-u, --url, --uuid TEXT URL or UUID of the manga +--read FILE Path of file with manga links to download. One per line + +verbosity: [mutually_exclusive] +--verbose Verbose logging. More log output [default: 20] +--lean Lean logging. Minimal log output [default: 20] +--debug Debug logging. Most log output [default: 20] + +-c, --chapters TEXT Chapters to download +-p, --path PATH Download path [default: downloads] +-l, --language TEXT Manga language [default: en] +--list List all available chapters +--format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz] +--forcevol Force naming of volumes. For mangas where chapters reset each volume +--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5] +--hook-manga-pre TEXT Commands to execute before the manga download starts +--hook-manga-post TEXT Commands to execute after the manga download finished +--hook-chapter-pre TEXT Commands to execute before the chapter download starts +--hook-chapter-post TEXT Commands to execute after the chapter download finished ``` ## Contribution / Bugs diff --git a/contrib/requirements_dev.txt b/contrib/requirements_dev.txt index 8f13f63..c019b6c 100644 --- a/contrib/requirements_dev.txt +++ b/contrib/requirements_dev.txt @@ -1,5 +1,9 @@ # application requirements -requests>=2.24.0 +requests>=2.28.0 +loguru>=0.6.0 +click>=8.1.3 +click-option-group>=0.5.5 + img2pdf>=0.4.4 # dev and testing requirements diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 index 7743ea1..2766215 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile.amd64 @@ -1,4 +1,4 @@ -FROM cr.44net.ch/baseimages/debian-s6:11.5.5-linux-amd64 +FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-amd64 # set version label ARG BUILD_VERSION diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 9bbb332..883a403 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,4 +1,4 @@ -FROM cr.44net.ch/baseimages/debian-s6:11.5.5-linux-arm64 +FROM cr.44net.ch/baseimages/debian-s6:11.5-linux-arm64 # set version label ARG BUILD_VERSION diff --git a/docker/README.md b/docker/README.md index 420a0b6..ff2c45d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,8 +1,10 @@ # Docker container of manga-dlp +> Full docs: https://manga-dlp.ivn.sh/docker + ## Quick start -> the pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64. +The pdf creation only works on amd64 images, as it unfortunately is incompatible with arm64. ```sh # with docker-compose @@ -13,135 +15,3 @@ docker-compose up -d # with docker run docker run -v ./downloads:/app/downloads -v ./mangas.txt:/app/mangas.txt olofvndrhr/manga-dlp ``` - -### Change UID/GID - -> The default UID and GID are 4444. - -You can change the UID and GID of the container user simply with: - -```yml -# docker-compose.yml -environment: - - PUID= - - PGID= -``` - -```sh -docker run -e PUID= -e PGID= -``` - -## Environment variables - -You can configure the default schedule via environment variables. Don't forget to set `MDLP_GENERATE_SCHEDULE` to "true" -, else -it will not generate it (it will just use the default one). - -For more info's about the options, you can look in the main scripts [README.md](../README.md) - -| ENV Variable | Default | manga-dlp option | Info | -|:-----------------------|:----------------|:-------------------------|--------------------------------------------------------------------------| -| MDLP_GENERATE_SCHEDULE | false | none | Has to be set to "true" to generate the config via environment variables | -| MDLP_PATH | /app/downloads | --path | | -| MDLP_READ | /app/mangas.txt | --read | | -| MDLP_LANGUAGE | en | --language | | -| MDLP_CHAPTERS | all | --chapter | | -| MDLP_FILE_FORMAT | cbz | --format | | -| MDLP_WAIT | 0.5 | --wait | | -| MDLP_FORCEVOL | false | --forcevol | | -| MDLP_LOG_LEVEL | lean | --lean/--verbose/--debug | Can either be set to: "lean", "verbose" or "debug" | - -## Run commands in container - -> You don't need to use the full path of manga-dlp.py because `/app` already is the working directory - -You can simply use the `docker exec` command to run the scripts like normal. - -```sh -docker exec python3 manga-dlp.py -``` - -## Run your own schedule - -The default config runs `manga-dlp.py` once a day at 12:00 and fetches every chapter of the mangas listed in the file -`mangas.txt` in the root directory of this repo. - -#### The default schedule: - -```sh -#!/bin/bash - -python3 /app/manga-dlp.py \ - --path /app/downloads \ - --read /app/mangas.txt \ - --chapters all \ - --wait 2 \ - --lean -``` - -To use your own schedule you need to mount (override) the default schedule or add new ones to the crontab. - -> Don't forget to add the cron entries for every new schedule - -```yml -# docker-compose.yml -volumes: - - ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab - - ./crontab2:/etc/cron.d/something # adds a new one crontab file - - ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule - - ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule -``` - -```sh -docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab -docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file -docker run -v ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule -docker run -v ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule -``` - -#### The default crontab file: - -```sh -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin - -# default crontab to run manga-dlp once a day -# and get all (new) chapters of the mangas in -# the file mangas.txt -# "/proc/1/fd/1 2>&1" is to show the logs in the container -# "s6-setuidgid abc" is used to set the permissions - -0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1 -``` - -## Add mangas to mangas.txt - -If you use the default crontab you still need to add some mangas to mangas.txt. This is done almost identical to adding -your own cron schedule. If you use a custom cron schedule you need to mount the file you specified with `--read`. - -```yml -# docker-compose.yml -volumes: - - ./mangas.txt:/app/mangas.txt -``` - -```sh -docker run -v ./mangas.txt:/app/mangas.txt -``` - -## Change download directory - -Per default as in the script, it downloads everything to "downloads" in the scripts root directory. This data does not -persist with container recreation, so you need to mount it. This is already done in the quick start section. If you want -to change the path of the host, simply change `./downloads/` to a path of your choice. - -```yml -# docker-compose.yml -volumes: - - ./downloads/:/app/downloads -``` - -```sh -docker run -v ./downloads/:/app/downloads -``` - diff --git a/docs/pages/hooks.md b/docs/pages/hooks.md index 4306099..3bf55eb 100644 --- a/docs/pages/hooks.md +++ b/docs/pages/hooks.md @@ -3,7 +3,7 @@ ## Available hooks You can run custom hooks with manga-dlp for specific events. -They are run with the `subproccess.call` function, so they get run directly by your operating system. +They are run with the `subproccess.run` function, so they get run directly by your operating system. The available hook events are: diff --git a/docs/pages/index.md b/docs/pages/index.md index 13df0e2..f8f7fed 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -87,30 +87,34 @@ See the docker [README](docker/) ## Options ```txt -usage: manga-dlp.py [-h] (-u URL_UUID | --read READ | -v) [-c CHAPTERS] [-p PATH] [-l LANG] [--list] [--format FORMAT] [--forcevol] [--wait WAIT] [--lean | --verbose | --debug] [--hook-manga-pre HOOK_MANGA_PRE] -[--hook-manga-post HOOK_MANGA_POST] [--hook-chapter-pre HOOK_CHAPTER_PRE] [--hook-chapter-post HOOK_CHAPTER_POST] +Usage: manga-dlp.py [OPTIONS] Script to download mangas from various sites -optional arguments: --h, --help show this help message and exit --u URL_UUID, --url URL_UUID, --uuid URL_UUID URL or UUID of the manga ---read READ Path of file with manga links to download. One per line --v, --version Show version of manga-dlp and exit --c CHAPTERS, --chapters CHAPTERS Chapters to download --p PATH, --path PATH Download path. Defaults to "/downloads" --l LANG, --language LANG Manga language. Defaults to "en" --> english ---list List all available chapters. Defaults to false ---format FORMAT Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz' ---forcevol Force naming of volumes. For mangas where chapters reset each volume ---wait WAIT Time to wait for each picture to download in seconds(float). Defaults 0.5 ---lean Lean logging. Minimal log output. Defaults to false ---verbose Verbose logging. More log output. Defaults to false ---debug Debug logging. Most log output. Defaults to false ---hook-manga-pre HOOK_MANGA_PRE Commands to execute before the manga download starts ---hook-manga-post HOOK_MANGA_POST Commands to execute after the manga download finished ---hook-chapter-pre HOOK_CHAPTER_PRE Commands to execute before the chapter download starts ---hook-chapter-post HOOK_CHAPTER_POST Commands to execute after the chapter download finished +Options: +--help Show this message and exit. +--version Show the version and exit. + +source: [mutually_exclusive, required] +-u, --url, --uuid TEXT URL or UUID of the manga +--read FILE Path of file with manga links to download. One per line + +verbosity: [mutually_exclusive] +--verbose Verbose logging. More log output [default: 20] +--lean Lean logging. Minimal log output [default: 20] +--debug Debug logging. Most log output [default: 20] + +-c, --chapters TEXT Chapters to download +-p, --path PATH Download path [default: downloads] +-l, --language TEXT Manga language [default: en] +--list List all available chapters +--format TEXT Archive format to create. An empty string means dont archive the folder [default: cbz] +--forcevol Force naming of volumes. For mangas where chapters reset each volume +--wait FLOAT Time to wait for each picture to download in seconds(float) [default: 0.5] +--hook-manga-pre TEXT Commands to execute before the manga download starts +--hook-manga-post TEXT Commands to execute after the manga download finished +--hook-chapter-pre TEXT Commands to execute before the chapter download starts +--hook-chapter-post TEXT Commands to execute after the chapter download finished ``` ## Contribution / Bugs diff --git a/manga-dlp.py b/manga-dlp.py index b880acf..211685f 100644 --- a/manga-dlp.py +++ b/manga-dlp.py @@ -1,4 +1,6 @@ -from mangadlp.input import main +import sys + +import mangadlp.cli if __name__ == "__main__": - main() + sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter diff --git a/mangadlp/__init__.py b/mangadlp/__init__.py index c2a631f..e69de29 100644 --- a/mangadlp/__init__.py +++ b/mangadlp/__init__.py @@ -1,4 +0,0 @@ -from mangadlp.logger import prepare_logger - -# prepare logger with default level INFO==20 -prepare_logger() diff --git a/mangadlp/__main__.py b/mangadlp/__main__.py index eb9209e..211685f 100644 --- a/mangadlp/__main__.py +++ b/mangadlp/__main__.py @@ -1,6 +1,6 @@ import sys -from mangadlp.input import main +import mangadlp.cli if __name__ == "__main__": - sys.exit(main()) + sys.exit(mangadlp.cli.main()) # pylint: disable=no-value-for-parameter diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index 8d63a04..c35f96f 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -3,12 +3,9 @@ import sys from time import sleep import requests +from loguru import logger as log from mangadlp import utils -from mangadlp.logger import Logger - -# prepare logger -log = Logger(__name__) class Mangadex: @@ -24,7 +21,7 @@ class Mangadex: api_name (str): Name of the API manga_uuid (str): UUID of the manga, without the url part manga_data (dict): Infos of the manga. Name, title etc - manga_title (str): The title of the manga, sanitized for all filesystems + manga_title (str): The title of the manga, sanitized for all file systems manga_chapter_data (dict): All chapter data of the manga. Volumes, chapters, chapter uuids and chapter names chapter_list (list): A list of all available chapters for the language @@ -57,7 +54,7 @@ class Mangadex: # make initial request def get_manga_data(self) -> dict: - log.verbose(f"Getting manga data for: {self.manga_uuid}") + log.debug(f"Getting manga data for: {self.manga_uuid}") counter = 1 while counter <= 3: try: @@ -98,7 +95,7 @@ class Mangadex: # get the title of the manga (and fix the filename) def get_manga_title(self) -> str: - log.verbose(f"Getting manga title for: {self.manga_uuid}") + log.debug(f"Getting manga title for: {self.manga_uuid}") manga_data = self.manga_data try: title = manga_data["data"]["attributes"]["title"][self.language] @@ -117,9 +114,7 @@ class Mangadex: # check if chapters are available in requested language def check_chapter_lang(self) -> int: - log.verbose( - f"Checking for chapters in specified language for: {self.manga_uuid}" - ) + log.debug(f"Checking for chapters in specified language for: {self.manga_uuid}") r = requests.get( f"{self.api_base_url}/manga/{self.manga_uuid}/feed?limit=0&{self.api_additions}" ) @@ -139,7 +134,7 @@ class Mangadex: # get chapter data like name, uuid etc def get_chapter_data(self) -> dict: - log.verbose(f"Getting chapter data for: {self.manga_uuid}") + log.debug(f"Getting chapter data for: {self.manga_uuid}") api_sorting = "order[chapter]=asc&order[volume]=asc" # check for chapters in specified lang total_chapters = self.check_chapter_lang() @@ -197,11 +192,11 @@ class Mangadex: # get images for the chapter (mangadex@home) def get_chapter_images(self, chapter: str, wait_time: float) -> list: - log.verbose(f"Getting chapter images for: {self.manga_uuid}") + log.debug(f"Getting chapter images for: {self.manga_uuid}") athome_url = f"{self.api_base_url}/at-home/server" chapter_uuid = self.manga_chapter_data[chapter][0] - # retry up to two times if the api applied ratelimits + # retry up to two times if the api applied rate limits api_error = False counter = 1 while counter <= 3: @@ -244,7 +239,7 @@ class Mangadex: # create list of chapters def create_chapter_list(self) -> list: - log.verbose(f"Creating chapter list for: {self.manga_uuid}") + log.debug(f"Creating chapter list for: {self.manga_uuid}") chapter_list = [] for chapter in self.manga_chapter_data.items(): chapter_info = self.get_chapter_infos(chapter[0]) diff --git a/mangadlp/app.py b/mangadlp/app.py index 24eb635..43889b6 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -4,13 +4,11 @@ import sys from pathlib import Path from typing import Any +from loguru import logger as log + from mangadlp import downloader, utils from mangadlp.api.mangadex import Mangadex -from mangadlp.hooks import Hooks -from mangadlp.logger import Logger - -# prepare logger -log = Logger(__name__) +from mangadlp.hooks import run_hook class MangaDLP: @@ -53,14 +51,12 @@ class MangaDLP: self.forcevol: bool = forcevol self.download_path: str = download_path self.download_wait: float = download_wait - # hooks - self.hook: Hooks = Hooks( - manga_pre_hook_cmd, - manga_post_hook_cmd, - chapter_pre_hook_cmd, - chapter_post_hook_cmd, - ) + self.manga_pre_hook_cmd: str = manga_pre_hook_cmd + self.manga_post_hook_cmd: str = manga_post_hook_cmd + self.chapter_pre_hook_cmd: str = chapter_pre_hook_cmd + self.chapter_post_hook_cmd: str = chapter_post_hook_cmd self.hook_infos: dict = {} + # prepare everything self._prepare() @@ -136,7 +132,7 @@ class MangaDLP: print_divider = "=========================================" # show infos log.info(f"{print_divider}") - log.lean(f"Manga Name: {self.manga_title}") + log.info(f"Manga Name: {self.manga_title}") log.info(f"Manga UUID: {self.manga_uuid}") log.info(f"Total chapters: {self.manga_total_chapters}") @@ -144,7 +140,7 @@ class MangaDLP: if self.list_chapters: log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}") log.info(f"{print_divider}\n") - sys.exit(0) + return # check chapters to download if not all if self.chapters.lower() == "all": @@ -155,7 +151,7 @@ class MangaDLP: ) # show chapters to download - log.lean(f"Chapters selected: {', '.join(chapters_to_download)}") + log.info(f"Chapters selected: {', '.join(chapters_to_download)}") log.info(f"{print_divider}") # create manga folder @@ -179,7 +175,12 @@ class MangaDLP: ) # start manga pre hook - self.hook.run("manga_pre", {"status": "starting"}, self.hook_infos) + run_hook( + command=self.manga_pre_hook_cmd, + hook_type="manga_pre", + status="starting", + **self.hook_infos, + ) # get chapters for chapter in chapters_to_download: @@ -201,24 +202,34 @@ class MangaDLP: log.info(f"Done with chapter '{chapter}'\n") # start chapter post hook - self.hook.run("chapter_post", {"status": "successful"}, self.hook_infos) + run_hook( + command=self.chapter_post_hook_cmd, + hook_type="chapter_post", + status="successful", + **self.hook_infos, + ) # done with manga log.info(f"{print_divider}") - log.lean(f"Done with manga: {self.manga_title}") + log.info(f"Done with manga: {self.manga_title}") # filter skipped list skipped_chapters = list(filter(None, skipped_chapters)) if len(skipped_chapters) >= 1: - log.lean(f"Skipped chapters: {', '.join(skipped_chapters)}") + log.info(f"Skipped chapters: {', '.join(skipped_chapters)}") # filter error list error_chapters = list(filter(None, error_chapters)) if len(error_chapters) >= 1: - log.lean(f"Chapters with errors: {', '.join(error_chapters)}") + log.info(f"Chapters with errors: {', '.join(error_chapters)}") # start manga post hook - self.hook.run("manga_post", {"status": "successful"}, self.hook_infos) + run_hook( + command=self.manga_post_hook_cmd, + hook_type="manga_post", + status="successful", + **self.hook_infos, + ) log.info(f"{print_divider}\n") @@ -242,8 +253,12 @@ class MangaDLP: f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}" ) - self.hook.run( - "chapter_pre", {"status": "skipped", "reason": "No images"}, {} + run_hook( + command=self.chapter_pre_hook_cmd, + hook_type="chapter_pre", + status="skipped", + reason="No images", + **self.hook_infos, ) # add to skipped chapters list @@ -271,8 +286,12 @@ class MangaDLP: if chapter_archive_path.exists(): log.info(f"'{chapter_archive_path}' already exists. Skipping") - self.hook.run( - "chapter_pre", {"status": "skipped", "reason": "Existing"}, {} + run_hook( + command=self.chapter_pre_hook_cmd, + hook_type="chapter_pre", + status="skipped", + reason="Existing", + **self.hook_infos, ) # add to skipped chapters list @@ -289,10 +308,10 @@ class MangaDLP: chapter_path.mkdir(parents=True, exist_ok=True) # verbose log - log.verbose(f"Chapter UUID: {chapter_infos['uuid']}") - log.verbose(f"Filename: '{chapter_archive_path.name}'") - log.verbose(f"File path: '{chapter_archive_path}'") - log.verbose(f"Image URLS:\n{chapter_image_urls}") + log.debug(f"Chapter UUID: {chapter_infos['uuid']}") + log.debug(f"Filename: '{chapter_archive_path.name}'") + log.debug(f"File path: '{chapter_archive_path}'") + log.debug(f"Image URLS:\n{chapter_image_urls}") # create dict with all variables for the hooks self.hook_infos.update( @@ -308,10 +327,15 @@ class MangaDLP: ) # start chapter pre hook - self.hook.run("chapter_pre", {"status": "starting"}, self.hook_infos) + run_hook( + command=self.chapter_pre_hook_cmd, + hook_type="chapter_pre", + status="starting", + **self.hook_infos, + ) # log - log.lean(f"Downloading: '{chapter_filename}'") + log.info(f"Downloading: '{chapter_filename}'") # download images try: @@ -324,8 +348,13 @@ class MangaDLP: except Exception: log.error(f"Cant download: '{chapter_filename}'. Skipping") - self.hook.run( - "chapter_post", {"status": "error", "reason": "Download error"}, {} + # run chapter post hook + run_hook( + command=self.chapter_post_hook_cmd, + hook_type="chapter_post", + status="starting", + reason="Download error", + **self.hook_infos, ) # add to skipped chapters list @@ -340,13 +369,13 @@ class MangaDLP: else: # Done with chapter - log.lean(f"Successfully downloaded: '{chapter_filename}'") + log.info(f"Successfully downloaded: '{chapter_filename}'") return {"chapter_path": chapter_path} # create an archive of the chapter if needed def archive_chapter(self, chapter_path: Path) -> dict: - log.lean(f"Creating archive '{chapter_path}{self.file_format}'") + log.info(f"Creating archive '{chapter_path}{self.file_format}'") try: # check if image folder is existing if not chapter_path.exists(): diff --git a/mangadlp/cli.py b/mangadlp/cli.py new file mode 100644 index 0000000..d78881a --- /dev/null +++ b/mangadlp/cli.py @@ -0,0 +1,240 @@ +from pathlib import Path + +import click +from click_option_group import ( + MutuallyExclusiveOptionGroup, + RequiredMutuallyExclusiveOptionGroup, + optgroup, +) +from loguru import logger as log + +from mangadlp import app +from mangadlp.__about__ import __version__ +from mangadlp.logger import prepare_logger + + +# read in the list of links from a file +def readin_list(_, __, value) -> list: + if not value: + return [] + + list_file = Path(value) + click.echo(f"Reading in file '{list_file}'") + try: + url_str = list_file.read_text(encoding="utf-8") + url_list = url_str.splitlines() + except Exception as exc: + raise click.BadParameter("Can't get links from the file") from exc + + # filter empty lines and remove them + filtered_list = list(filter(len, url_list)) + log.info(f"Mangas from list: {filtered_list}") + + return filtered_list + + +@click.command(context_settings={"max_content_width": 150}) +@click.help_option() +@click.version_option(version=__version__, package_name="manga-dlp") +@optgroup.group("source", cls=RequiredMutuallyExclusiveOptionGroup) +@optgroup.option( + "-u", + "--url", + "--uuid", + "url_uuid", + type=str, + default=None, + show_default=True, + help="URL or UUID of the manga", +) +@optgroup.option( + "--read", + "read_mangas", + is_eager=True, + callback=readin_list, + type=click.Path(exists=True, dir_okay=False), + default=None, + show_default=True, + help="Path of file with manga links to download. One per line", +) +@optgroup.group("verbosity", cls=MutuallyExclusiveOptionGroup) +@optgroup.option( + "--verbose", + "verbosity", + flag_value=20, + default=20, + show_default=True, + help="Verbose logging. More log output", +) +@optgroup.option( + "--lean", + "verbosity", + flag_value=25, + default=20, + show_default=True, + help="Lean logging. Minimal log output", +) +@optgroup.option( + "--debug", + "verbosity", + flag_value=10, + default=20, + show_default=True, + help="Debug logging. Most log output", +) +@click.option( + "-c", + "--chapters", + "chapters", + type=str, + default=None, + required=False, + show_default=True, + help="Chapters to download", +) +@click.option( + "-p", + "--path", + "path", + type=click.Path(exists=False), + default="downloads", + required=False, + show_default=True, + help="Download path", +) +@click.option( + "-l", + "--language", + "lang", + type=str, + default="en", + required=False, + show_default=True, + help="Manga language", +) +@click.option( + "--list", + "list_chapters", + is_flag=True, + default=False, + required=False, + show_default=True, + help="List all available chapters", +) +@click.option( + "--format", + "chapter_format", + type=str, + default="cbz", + required=False, + show_default=True, + help="Archive format to create. An empty string means dont archive the folder", +) +@click.option( + "--forcevol", + "forcevol", + is_flag=True, + default=False, + required=False, + show_default=True, + help="Force naming of volumes. For mangas where chapters reset each volume", +) +@click.option( + "--wait", + "wait_time", + type=float, + default=0.5, + required=False, + show_default=True, + help="Time to wait for each picture to download in seconds(float)", +) +# hook options +@click.option( + "--hook-manga-pre", + "hook_manga_pre", + type=str, + default=None, + required=False, + show_default=True, + help="Commands to execute before the manga download starts", +) +@click.option( + "--hook-manga-post", + "hook_manga_post", + type=str, + default=None, + required=False, + show_default=True, + help="Commands to execute after the manga download finished", +) +@click.option( + "--hook-chapter-pre", + "hook_chapter_pre", + type=str, + default=None, + required=False, + show_default=True, + help="Commands to execute before the chapter download starts", +) +@click.option( + "--hook-chapter-post", + "hook_chapter_post", + type=str, + default=None, + required=False, + show_default=True, + help="Commands to execute after the chapter download finished", +) +@click.pass_context +def main( + ctx: click.Context, + url_uuid: str, + read_mangas: list, + verbosity: int, + chapters: str, + path: str, + lang: str, + list_chapters: bool, + chapter_format: str, + forcevol: bool, + wait_time: float, + hook_manga_pre: str, + hook_manga_post: str, + hook_chapter_pre: str, + hook_chapter_post: str, +): # pylint: disable=too-many-locals + + """ + Script to download mangas from various sites + + """ + + # set loglevel and log format + prepare_logger(verbosity) + + # list all params + log.debug(ctx.params) + + # all request mangas + requested_mangas = [url_uuid] if url_uuid else read_mangas + + for manga in requested_mangas: + mdlp = app.MangaDLP( + url_uuid=manga, + language=lang, + chapters=chapters, + list_chapters=list_chapters, + file_format=chapter_format, + forcevol=forcevol, + download_path=path, + download_wait=wait_time, + manga_pre_hook_cmd=hook_manga_pre, + manga_post_hook_cmd=hook_manga_post, + chapter_pre_hook_cmd=hook_chapter_pre, + chapter_post_hook_cmd=hook_chapter_post, + ) + mdlp.get_manga() + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index 6495cc1..7fde1e5 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -6,12 +6,9 @@ from time import sleep from typing import Union import requests +from loguru import logger as log from mangadlp import utils -from mangadlp.logger import Logger - -# prepare logger -log = Logger(__name__) # download images @@ -29,7 +26,7 @@ def download_chapter( # show progress bar for default log level if logging.root.level == logging.INFO: utils.progress_bar(image_num, total_img) - log.verbose(f"Downloading image {image_num}/{total_img}") + log.debug(f"Downloading image {image_num}/{total_img}") counter = 1 while counter <= 3: diff --git a/mangadlp/hooks.py b/mangadlp/hooks.py index 706c988..31c702a 100644 --- a/mangadlp/hooks.py +++ b/mangadlp/hooks.py @@ -1,71 +1,40 @@ import os import subprocess -from mangadlp.logger import Logger - -# prepare logger -log = Logger(__name__) +from loguru import logger as log -class Hooks: - """Pre- and post-hooks for each download. - +def run_hook(command: str, hook_type: str, **kwargs) -> int: + """ Args: - cmd_manga_pre (str): Commands to execute before the manga download starts - cmd_manga_post (str): Commands to execute after the manga download finished - cmd_chapter_pre (str): Commands to execute before the chapter download starts - cmd_chapter_post (str): Commands to execute after the chapter download finished + command (str): command to run + hook_type (str): type of the hook + kwargs: key value pairs of env vars to set + Returns: + exit_code (int): exit code of command """ - def __init__( - self, - cmd_manga_pre: str, - cmd_manga_post: str, - cmd_chapter_pre: str, - cmd_chapter_post: str, - ) -> None: - self.cmd_manga_pre = cmd_manga_pre - self.cmd_manga_post = cmd_manga_post - self.cmd_chapter_pre = cmd_chapter_pre - self.cmd_chapter_post = cmd_chapter_post + # check if hook commands are empty + if not command or command == "None": + log.debug(f"Hook '{hook_type}' empty. Not running") + return 2 - def run(self, hook_type: str, hook_status: dict, hook_info: dict) -> int: - if hook_type == "manga_pre": - hook_cmd_str = self.cmd_manga_pre - elif hook_type == "manga_post": - hook_cmd_str = self.cmd_manga_post - elif hook_type == "chapter_pre": - hook_cmd_str = self.cmd_chapter_pre - elif hook_type == "chapter_post": - hook_cmd_str = self.cmd_chapter_post - else: - log.error(f"Hook type '{hook_type}' is not valid. Not running") - return 1 + command_list = command.split(" ") - # check if hook commands are empty - if not hook_cmd_str or hook_cmd_str == "None": - log.verbose(f"Hook '{hook_type}' empty. Not running") - return 2 + # setting env vars + for key, value in kwargs.items(): + os.environ[f"MDLP_{key.upper()}"] = str(value) - hook_cmd_list = hook_cmd_str.split(" ") + # running command + log.info(f"Hook '{hook_type}' - running command: '{command}'") + proc = subprocess.run(command_list, check=False, timeout=15, encoding="utf8") + exit_code = proc.returncode - # setting env vars - hook_info["hook_type"] = hook_type - hook_info["status"] = hook_status.get("status") - hook_info["reason"] = hook_status.get("reason") + if exit_code == 0: + log.debug("Hook returned status code 0. All good") + else: + log.warning(f"Hook returned status code {exit_code}. Possible error") - for key, value in hook_info.items(): - os.environ[f"MDLP_{key.upper()}"] = str(value) - - # running command - log.info(f"Hook '{hook_type}' - running command: '{hook_cmd_str}'") - ecode = subprocess.call(hook_cmd_list) - - if ecode == 0: - log.verbose("Hook returned status code 0. All good") - else: - log.warning(f"Hook returned status code {ecode}. Possible error") - - # return exit code of command - return ecode + # return exit code of command + return exit_code diff --git a/mangadlp/input.py b/mangadlp/input.py deleted file mode 100644 index 9eef6c3..0000000 --- a/mangadlp/input.py +++ /dev/null @@ -1,268 +0,0 @@ -import argparse -import sys -from pathlib import Path - -from mangadlp import app, logger -from mangadlp.__about__ import __version__ -from mangadlp.logger import Logger - -# prepare logger -log = Logger(__name__) - - -def check_args(args): - # set logger formatting - logger.format_logger(args.verbosity) - # check if --version was used - if args.version: - print(f"manga-dlp version: {__version__}") - sys.exit(0) - # check if a readin list was provided - if not args.read: - # single manga, no readin list - call_app(args) - else: - # multiple mangas - url_list = readin_list(args.read) - for url in url_list: - args.url_uuid = url - call_app(args) - - -# read in the list of links from a file -def readin_list(readlist: str) -> list: - list_file = Path(readlist) - log.verbose(f"Reading in list '{str(list_file)}'") - try: - url_str = list_file.read_text(encoding="utf-8") - url_list = url_str.splitlines() - except Exception as exc: - raise IOError from exc - - # filter empty lines and remove them - filtered_list = list(filter(len, url_list)) - log.verbose(f"Mangas from list: {filtered_list}") - - return filtered_list - - -def call_app(args): - # call main function with all input arguments - mdlp = app.MangaDLP( - url_uuid=args.url_uuid, - language=args.lang, - chapters=args.chapters, - list_chapters=args.list, - file_format=args.format, - forcevol=args.forcevol, - download_path=args.path, - download_wait=args.wait, - manga_pre_hook_cmd=args.hook_manga_pre, - manga_post_hook_cmd=args.hook_manga_post, - chapter_pre_hook_cmd=args.hook_chapter_pre, - chapter_post_hook_cmd=args.hook_chapter_post, - ) - mdlp.get_manga() - - -def get_input(): - print(f"manga-dlp version: {__version__}") - print("Enter details of the manga you want to download:") - while True: - try: - url_uuid = str(input("Url or UUID: ")) - readlist = str(input("List with links (optional): ")) - language = str(input("Language: ")) or "en" - list_chapters = str(input("List chapters? y/N: ")) - if list_chapters.lower() in {"y", "yes"}: - chapters = str(input("Chapters: ")) - except KeyboardInterrupt: - sys.exit(1) - except Exception: - continue - else: - break - - args = [ - "-l", - language, - "-c", - chapters, - ] - if url_uuid: - args.extend(("-u", url_uuid)) - if readlist: - args.extend(("--read", readlist)) - if list_chapters.lower() in {"y", "yes"}: - args.append("--list") - - # start script again with the arguments - sys.argv.extend(args) - log.info(f"Args: {sys.argv}") - get_args() - - -def get_args(): - parser = argparse.ArgumentParser( - description="Script to download mangas from various sites" - ) - action = parser.add_mutually_exclusive_group(required=True) - verbosity = parser.add_mutually_exclusive_group(required=False) - - # selection options - action.add_argument( - "-u", - "--url", - "--uuid", - dest="url_uuid", - required=False, - help="URL or UUID of the manga", - action="store", - ) - action.add_argument( - "--read", - dest="read", - required=False, - help="Path of file with manga links to download. One per line", - action="store", - ) - action.add_argument( - "-v", - "--version", - dest="version", - required=False, - help="Show version of manga-dlp and exit", - action="store_true", - ) - - # base options - parser.add_argument( - "-c", - "--chapters", - dest="chapters", - required=False, - help="Chapters to download", - action="store", - ) - parser.add_argument( - "-p", - "--path", - dest="path", - required=False, - help='Download path. Defaults to "/downloads"', - action="store", - default="downloads", - ) - parser.add_argument( - "-l", - "--language", - dest="lang", - required=False, - help='Manga language. Defaults to "en" --> english', - action="store", - default="en", - ) - parser.add_argument( - "--list", - dest="list", - required=False, - help="List all available chapters. Defaults to false", - action="store_true", - ) - parser.add_argument( - "--format", - dest="format", - required=False, - help="Archive format to create. An empty string means dont archive the folder. Defaults to 'cbz'", - action="store", - default="cbz", - ) - parser.add_argument( - "--forcevol", - dest="forcevol", - required=False, - help="Force naming of volumes. For mangas where chapters reset each volume", - action="store_true", - ) - parser.add_argument( - "--wait", - dest="wait", - required=False, - type=float, - default=0.5, - help="Time to wait for each picture to download in seconds(float). Defaults 0.5", - ) - - # logging options - verbosity.add_argument( - "--lean", - dest="verbosity", - required=False, - help="Lean logging. Minimal log output. Defaults to false", - action="store_const", - const=25, - default=20, - ) - verbosity.add_argument( - "--verbose", - dest="verbosity", - required=False, - help="Verbose logging. More log output. Defaults to false", - action="store_const", - const=15, - default=20, - ) - verbosity.add_argument( - "--debug", - dest="verbosity", - required=False, - help="Debug logging. Most log output. Defaults to false", - action="store_const", - const=10, - default=20, - ) - # hook options - parser.add_argument( - "--hook-manga-pre", - dest="hook_manga_pre", - required=False, - help="Commands to execute before the manga download starts", - action="store", - ) - parser.add_argument( - "--hook-manga-post", - dest="hook_manga_post", - required=False, - help="Commands to execute after the manga download finished", - action="store", - ) - parser.add_argument( - "--hook-chapter-pre", - dest="hook_chapter_pre", - required=False, - help="Commands to execute before the chapter download starts", - action="store", - ) - parser.add_argument( - "--hook-chapter-post", - dest="hook_chapter_post", - required=False, - help="Commands to execute after the chapter download finished", - action="store", - ) - - # parser.print_help() - args = parser.parse_args() - - check_args(args) - - -def main(): - if len(sys.argv) > 1: - get_args() - else: - get_input() - - -if __name__ == "__main__": - main() diff --git a/mangadlp/logger.py b/mangadlp/logger.py index 17c1910..590248e 100644 --- a/mangadlp/logger.py +++ b/mangadlp/logger.py @@ -1,71 +1,35 @@ import logging +import sys -DATE_FMT = "%Y-%m-%dT%H:%M:%S%z" +from loguru import logger + +LOGGING_FMT: str = ( + "%(asctime)s | (D) [%(levelname)-7s] [%(name)-10s] [%(funcName)-20s]: %(message)s" +) +LOGURU_FMT: str = "{time:%Y-%m-%dT%H:%M:%S%z} | (C) [{level: <7}] [{name: <10}] [{function: <20}]: {message}" -# prepare custom levels and default config of logger -def prepare_logger(): +def enable_default_logger(loglevel: int) -> None: + logging.root.handlers = [] + logging.basicConfig( - format="%(asctime)s | [%(levelname)s] [%(name)s]: %(message)s", - datefmt=DATE_FMT, - level=20, + format=LOGGING_FMT, + datefmt="%Y-%m-%dT%H:%M:%S%z", + level=loglevel, handlers=[logging.StreamHandler()], ) - logging.addLevelName(level=15, levelName="VERBOSE") - logging.addLevelName(level=25, levelName="LEAN") -# set log message format -def format_logger(verbosity: int): - logging.getLogger().setLevel(verbosity) - - # dont show log level name on default/lean logging - if verbosity >= 20: - logging.basicConfig( - format="%(asctime)s | %(message)s", - datefmt=DATE_FMT, - force=True, - ) - else: - logging.basicConfig( - format="%(asctime)s | [%(levelname)s] [%(name)s]: %(message)s", - datefmt=DATE_FMT, - force=True, - ) - - -class Logger: - """Default logger for manga-dlp. - - Args: - name (str): Name of the logger - - """ - - def __init__(self, name: str): - self.name = name - # create logger - self.log = logging.getLogger(self.name) - - # custom log levels - def verbose(self, message: str): - self.log.log(level=15, msg=message) - - def lean(self, message: str): - self.log.log(level=25, msg=message) - - # default log levels - def critical(self, message: str): - self.log.critical(msg=message) - - def error(self, message: str): - self.log.error(msg=message) - - def warning(self, message: str): - self.log.warning(msg=message) - - def info(self, message: str): - self.log.info(msg=message) - - def debug(self, message: str): - self.log.debug(msg=message) +# create config for a normal stderr logger +def prepare_logger(loglevel: int) -> None: + config: dict = { + "handlers": [ + { + "sink": sys.stdout, + "level": loglevel, + "format": LOGURU_FMT, + }, + ], + } + logger.configure(**config) + enable_default_logger(loglevel) diff --git a/mangadlp/utils.py b/mangadlp/utils.py index 960a600..fd3f8ab 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -4,15 +4,12 @@ from pathlib import Path from typing import Any from zipfile import ZipFile -from mangadlp.logger import Logger - -# prepare logger -log = Logger(__name__) +from loguru import logger as log # create an archive of the chapter images def make_archive(chapter_path: Path, file_format: str) -> None: - zip_path = Path(f"{chapter_path}.zip") + zip_path: Path = Path(f"{chapter_path}.zip") try: # create zip with ZipFile(zip_path, "w") as zipfile: @@ -26,13 +23,13 @@ def make_archive(chapter_path: Path, file_format: str) -> None: def make_pdf(chapter_path: Path) -> None: try: - import img2pdf + import img2pdf # pylint: disable=import-outside-toplevel except Exception as exc: log.error("Cant import img2pdf. Please install it first") raise ImportError from exc - pdf_path = Path(f"{chapter_path}.pdf") - images = [] + pdf_path: Path = Path(f"{chapter_path}.pdf") + images: list[str] = [] for file in chapter_path.iterdir(): images.append(str(file)) try: @@ -85,6 +82,9 @@ def get_chapter_list(chapters: str, available_chapters: list) -> list: # remove illegal characters etc def fix_name(filename: str) -> str: + filename = filename.encode(encoding="ascii", errors="ignore").decode( + encoding="utf8" + ) # remove illegal characters filename = re.sub(r'[/\\<>:;|?*!@"]', "", filename) # remove multiple dots diff --git a/pyproject.toml b/pyproject.toml index 5c37d3d..ecef4c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,10 @@ classifiers = [ "Programming Language :: Python :: 3.10", ] dependencies = [ - "requests>=2.24.0", + "requests>=2.28.0", + "loguru>=0.6.0", + "click>=8.1.3", + "click-option-group>=0.5.5", ] [project.urls] @@ -36,8 +39,8 @@ Tracker = "https://github.com/olofvndrhr/manga-dlp/issues" Source = "https://github.com/olofvndrhr/manga-dlp" [project.scripts] -mangadlp = "mangadlp.input:main" -manga-dlp = "mangadlp.input:main" +mangadlp = "mangadlp.cli:main" +manga-dlp = "mangadlp.cli:main" [tool.hatch.version] path = "mangadlp/__about__.py" @@ -53,6 +56,10 @@ packages = ["mangadlp"] [tool.hatch.envs.default] dependencies = [ + "requests>=2.28.0", + "loguru>=0.6.0", + "click>=8.1.3", + "click-option-group>=0.5.5", "img2pdf>=0.4.4", "hatch>=1.2.1", "hatchling>=1.4.1", @@ -121,7 +128,7 @@ ignore_errors = true py-version = "3.9" [tool.pylint.logging] -logging-modules = ["logging"] -disable = "C0301, C0114, C0116, W0703, R0902, R0913" +logging-modules = ["logging", "loguru"] +disable = "C0301, C0114, C0116, W0703, R0902, R0913, E0401, W1203" good-names = "r" -#logging-format-style = "fstr" +logging-format-style = "new" diff --git a/requirements.txt b/requirements.txt index 122a4cb..0fe1f62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -requests>=2.24.0 +requests>=2.28.0 +loguru>=0.6.0 +click>=8.1.3 +click-option-group>=0.5.5 img2pdf>=0.4.4 diff --git a/tests/test_04_input.py b/tests/test_04_input.py index 2ab3f81..fb41fe4 100644 --- a/tests/test_04_input.py +++ b/tests/test_04_input.py @@ -3,7 +3,7 @@ from pathlib import Path import pytest -import mangadlp.input as mdlpinput +import mangadlp.cli as mdlpinput def test_read_and_url(): @@ -54,7 +54,7 @@ def test_no_volume(): def test_readin_list(): list_file = "tests/test_list.txt" - test_list = mdlpinput.readin_list(list_file) + test_list = mdlpinput.readin_list(None, None, list_file) assert test_list == [ "https://mangadex.org/title/a96676e5-8ae2-425e-b549-7f15dd34a6d8/komi-san-wa-komyushou-desu",