diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c54a2bf..6015d27 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1.10 +current_version = 2.1.11 commit = True tag = False serialize = {major}.{minor}.{patch} diff --git a/.woodpecker/test_docker.yml b/.woodpecker/test_docker.yml index a1416ad..8aea5fa 100644 --- a/.woodpecker/test_docker.yml +++ b/.woodpecker/test_docker.yml @@ -30,7 +30,7 @@ pipeline: dockerfile: docker/Dockerfile.amd64 auto_tag: true auto_tag_suffix: linux-amd64-test - build_args: BUILD_VERSION=2.1.10 + build_args: BUILD_VERSION=2.1.11 # build docker image for arm64 test-build-arm64: @@ -46,4 +46,4 @@ pipeline: dockerfile: docker/Dockerfile.arm64 auto_tag: true auto_tag_suffix: linux-arm64-test - build_args: BUILD_VERSION=2.1.10 + build_args: BUILD_VERSION=2.1.11 diff --git a/.woodpecker/test_release.yml b/.woodpecker/test_release.yml index 0ebaa00..e3e434a 100644 --- a/.woodpecker/test_release.yml +++ b/.woodpecker/test_release.yml @@ -34,5 +34,5 @@ pipeline: image: cr.44net.ch/baseimages/debian-base pull: true commands: - - bash get_release_notes.sh 2.1.10 + - bash get_release_notes.sh 2.1.11 - cat RELEASENOTES.md diff --git a/.woodpecker/tests.yml b/.woodpecker/tests.yml index 30651e9..ab9e557 100644 --- a/.woodpecker/tests.yml +++ b/.woodpecker/tests.yml @@ -31,6 +31,14 @@ pipeline: commands: - isort --check-only --diff . + # check unused and missing imports + test-autoflake: + image: cr.44net.ch/ci-plugins/tests + pull: true + commands: + - autoflake --remove-all-unused-imports -r -v mangadlp/ + - autoflake --check --remove-all-unused-imports -r -v mangadlp/ + # check static typing - python test-mypy: image: cr.44net.ch/ci-plugins/tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 196e60a..e9bb014 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,30 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Add support for more sites +## [2.1.11] - 2022-07-18 + +### Fixed + +- The `--read` option now filters empty lines, so it will not generate an error anymore +- An error which was caused by the interactive input method when you did not specify a chapter or to list them +- Some typos + +### Added + +- Options to configure the default schedule in the docker container via environment variables +- Section the the docker [README.md](docker/README.md) for the new environment variables +- `autoflake` test in `justfile` +- Some more things which get logged + +### Changed + +- **BREAKING**: renamed the default schedule from `daily` to `daily.sh`. Don't forget to fix your bind-mounts to + overwrite + the default schedule +- Added the `.sh` suffix to the s6 init scripts for better compatibility +- Adjusted the new logging implementation. It shows now more info about the module the log is from, and some other + improvements + ## [2.1.10] - 2022-07-14 ### Fixed diff --git a/contrib/requirements_dev.txt b/contrib/requirements_dev.txt index af5b4d3..4b3cefb 100644 --- a/contrib/requirements_dev.txt +++ b/contrib/requirements_dev.txt @@ -11,3 +11,4 @@ pylint>=2.13.0 mypy>=0.940 tox>=3.24.5 hatch>=1.0.0 +autoflake>=1.4 diff --git a/docker/Dockerfile.amd64 b/docker/Dockerfile.amd64 index 775673a..8f98100 100644 --- a/docker/Dockerfile.amd64 +++ b/docker/Dockerfile.amd64 @@ -19,7 +19,9 @@ RUN \ # prepare app RUN \ echo "**** creating folders ****" \ - && mkdir -p /app + && mkdir -p /app \ + && echo "**** updating pip ****" \ + && python3 -m pip install --upgrade pip # cleanup installation RUN \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index ce20e2c..679c21f 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -19,7 +19,9 @@ RUN \ # prepare app RUN \ echo "**** creating folders ****" \ - && mkdir -p /app + && mkdir -p /app \ + && echo "**** updating pip ****" \ + && python3 -m pip install --upgrade pip # cleanup installation RUN \ diff --git a/docker/README.md b/docker/README.md index 5ddf6aa..420a0b6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -31,6 +31,26 @@ environment: docker run -e PUID= -e PGID= ``` +## Environment variables + +You can configure the default schedule via environment variables. Don't forget to set `MDLP_GENERATE_SCHEDULE` to "true" +, else +it will not generate it (it will just use the default one). + +For more info's about the options, you can look in the main scripts [README.md](../README.md) + +| ENV Variable | Default | manga-dlp option | Info | +|:-----------------------|:----------------|:-------------------------|--------------------------------------------------------------------------| +| MDLP_GENERATE_SCHEDULE | false | none | Has to be set to "true" to generate the config via environment variables | +| MDLP_PATH | /app/downloads | --path | | +| MDLP_READ | /app/mangas.txt | --read | | +| MDLP_LANGUAGE | en | --language | | +| MDLP_CHAPTERS | all | --chapter | | +| MDLP_FILE_FORMAT | cbz | --format | | +| MDLP_WAIT | 0.5 | --wait | | +| MDLP_FORCEVOL | false | --forcevol | | +| MDLP_LOG_LEVEL | lean | --lean/--verbose/--debug | Can either be set to: "lean", "verbose" or "debug" | + ## Run commands in container > You don't need to use the full path of manga-dlp.py because `/app` already is the working directory @@ -68,15 +88,15 @@ To use your own schedule you need to mount (override) the default schedule or ad volumes: - ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab - ./crontab2:/etc/cron.d/something # adds a new one crontab file - - ./schedule1:/app/schedules/daily # overwrites the default schedule - - ./schedule2:/app/schedules/weekly # adds a new schedule + - ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule + - ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule ``` ```sh docker run -v ./crontab:/etc/cron.d/mangadlp # overwrites the default crontab docker run -v ./crontab2:/etc/cron.d/something # adds a new one crontab file -docker run -v ./schedule1:/app/schedules/daily # overwrites the default schedule -docker run -v ./schedule2:/app/schedules/weekly # adds a new schedule +docker run -v ./schedule1.sh:/app/schedules/daily.sh # overwrites the default schedule +docker run -v ./schedule2.sh:/app/schedules/weekly.sh # adds a new schedule ``` #### The default crontab file: @@ -91,7 +111,7 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin # "/proc/1/fd/1 2>&1" is to show the logs in the container # "s6-setuidgid abc" is used to set the permissions -0 12 * * * root s6-setuidgid abc /app/schedules/daily > /proc/1/fd/1 2>&1 +0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1 ``` ## Add mangas to mangas.txt diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1b47ccb..db1dd21 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -13,15 +13,13 @@ services: - ./downloads/:/app/downloads/ # default manga download directory - ./mangas.txt:/app/mangas.txt # default file for manga links to download #- ./crontab:/etc/cron.d/mangadlp # path to default crontab - #- ./schedule:/app/schedules/daily # path to the default schedule which is run daily + #- ./schedule.sh:/app/schedules/daily.sh # path to the default schedule which is run daily environment: - TZ=Europe/Zurich - # - PUID= # custom userid - defaults to 4444 - # - PGID= # custom groupid - defaults to 4444 - + #- PUID= # custom user id - defaults to 4444 + #- PGID= # custom group id - defaults to 4444 networks: appnet: name: mangadlp driver: bridge - diff --git a/docker/rootfs/app/schedules/daily b/docker/rootfs/app/schedules/daily.sh similarity index 100% rename from docker/rootfs/app/schedules/daily rename to docker/rootfs/app/schedules/daily.sh diff --git a/docker/rootfs/etc/cont-init.d/20-setenv b/docker/rootfs/etc/cont-init.d/20-setenv.sh similarity index 60% rename from docker/rootfs/etc/cont-init.d/20-setenv rename to docker/rootfs/etc/cont-init.d/20-setenv.sh index 0062f63..7bc3f29 100644 --- a/docker/rootfs/etc/cont-init.d/20-setenv +++ b/docker/rootfs/etc/cont-init.d/20-setenv.sh @@ -4,8 +4,12 @@ # set all env variables for further use. If variable is unset, it will have the defaults on the right side after ":=" # custom env vars +: "${MDLP_GENERATE_SCHEDULE:=false}" +: "${MDLP_PATH:=/app/downloads}" +: "${MDLP_READ:=/app/mangas.txt}" : "${MDLP_LANGUAGE:=en}" -: "${MDLP_FORCEVOL:=false}" +: "${MDLP_CHAPTERS:=all}" : "${MDLP_FILE_FORMAT:=cbz}" -: "${MDLP_DOWNLOAD_WAIT:=2}" -: "${MDLP_VERBOSE:=false}" +: "${MDLP_WAIT:=0.5}" +: "${MDLP_FORCEVOL:=false}" +: "${MDLP_LOG_LEVEL:=lean}" diff --git a/docker/rootfs/etc/cont-init.d/51-fix-schedule.sh b/docker/rootfs/etc/cont-init.d/51-fix-schedule.sh new file mode 100644 index 0000000..097199d --- /dev/null +++ b/docker/rootfs/etc/cont-init.d/51-fix-schedule.sh @@ -0,0 +1,28 @@ +#!/usr/bin/with-contenv bash +# shellcheck shell=bash + +# source env variables +source /etc/cont-init.d/20-setenv.sh + +# check schedule +[[ -f "/app/schedules/daily.sh" ]] && DAILYSH=true +[[ -f "/app/schedules/daily" ]] && DAILY=true +# check crontab +if grep -q -e "/app/schedules/daily.sh\s" /etc/cron.d/mangadlp; then + CRONSH=true +elif grep -q -e "/app/schedules/daily\s" /etc/cron.d/mangadlp; then + CRON=true +fi + +# fix new .sh schedule if its not synced with the crontab +if [[ "${CRONSH}" == "true" ]] && [[ "${DAILYSH}" != "true" ]]; then + echo "Fixing new .sh schedule" + if ! ln -s /app/schedule/daily /app/schedule/daily.sh; then + echo "Cant fix schedule. Maybe the file is missing." + fi +elif [[ "${CRON}" == "true" ]] && [[ "${DAILY}" != "true" ]]; then + echo "Fixing new .sh schedule" + if ! ln -s /app/schedule/daily.sh /app/schedule/daily; then + echo "Cant fix schedule. Maybe the file is missing." + fi +fi diff --git a/docker/rootfs/etc/cont-init.d/52-set-schedule.sh b/docker/rootfs/etc/cont-init.d/52-set-schedule.sh new file mode 100644 index 0000000..4fbbe76 --- /dev/null +++ b/docker/rootfs/etc/cont-init.d/52-set-schedule.sh @@ -0,0 +1,60 @@ +#!/usr/bin/with-contenv bash +# shellcheck shell=bash + +# source env variables +source /etc/cont-init.d/20-setenv.sh + +function prepare_vars() { + # set log level + case "${MDLP_LOG_LEVEL}" in + "lean") + MDLP_LOG_LEVEL_FLAG=" --lean" + ;; + "verbose") + MDLP_LOG_LEVEL_FLAG=" --verbose" + ;; + "debug") + MDLP_LOG_LEVEL_FLAG=" --debug" + ;; + esac + + # check if forcevol should be used + if [[ "${MDLP_FORCEVOL,,}" == "true" ]]; then + # add backslash if log level is also specified + if [[ -n "${MDLP_LOG_LEVEL_FLAG}" ]]; then + MDLP_FORCEVOL_FLAG="\n --forcevol \\" + else + MDLP_FORCEVOL_FLAG="\n --forcevol" + fi + fi +} + +# set schedule with env variables +function set_vars() { + echo -ne "#!/bin/bash\n +python3 /app/manga-dlp.py \\ + --path ${MDLP_PATH} \\ + --read ${MDLP_READ} \\ + --language ${MDLP_LANGUAGE} \\ + --chapters ${MDLP_CHAPTERS} \\ + --format ${MDLP_FILE_FORMAT} \\ + --wait ${MDLP_WAIT}" \ + > /app/schedules/daily.sh + + # set forcevol or log level if specified + if [[ -n "${MDLP_FORCEVOL_FLAG}" ]] || [[ -n "${MDLP_LOG_LEVEL_FLAG}" ]]; then + sed -i 's/--wait '"${MDLP_WAIT}"'/--wait '"${MDLP_WAIT}"' \\/g' /app/schedules/daily.sh + echo -e "${MDLP_FORCEVOL_FLAG:-}" >> /app/schedules/daily.sh + echo -e "${MDLP_LOG_LEVEL_FLAG:-}" >> /app/schedules/daily.sh + else + # add final newline of not added before + echo -ne "\n" >> /app/schedules/daily.sh + fi +} + +# check if schedule should be generated +if [[ "${MDLP_GENERATE_SCHEDULE,,}" == "true" ]]; then + echo "Generating schedule" + prepare_vars + set_vars +fi diff --git a/docker/rootfs/etc/cont-init.d/80-fix-perms b/docker/rootfs/etc/cont-init.d/80-fix-perms.sh similarity index 93% rename from docker/rootfs/etc/cont-init.d/80-fix-perms rename to docker/rootfs/etc/cont-init.d/80-fix-perms.sh index bddecbc..e535c70 100644 --- a/docker/rootfs/etc/cont-init.d/80-fix-perms +++ b/docker/rootfs/etc/cont-init.d/80-fix-perms.sh @@ -2,7 +2,7 @@ # shellcheck shell=bash # source env variables -source /etc/cont-init.d/20-setenv +source /etc/cont-init.d/20-setenv.sh # fix permissions find '/app' -type 'd' \( -not -perm 775 -and -not -path '/app/downloads*' \) -exec chmod 775 '{}' \+ diff --git a/docker/rootfs/etc/cron.d/mangadlp b/docker/rootfs/etc/cron.d/mangadlp index 3db1d4d..40bfc62 100644 --- a/docker/rootfs/etc/cron.d/mangadlp +++ b/docker/rootfs/etc/cron.d/mangadlp @@ -7,5 +7,5 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin # "/proc/1/fd/1 2>&1" is to show the logs in the container # "s6-setuidgid abc" is used to set the permissions -0 12 * * * root s6-setuidgid abc /app/schedules/daily > /proc/1/fd/1 2>&1 +0 12 * * * root s6-setuidgid abc /app/schedules/daily.sh > /proc/1/fd/1 2>&1 diff --git a/justfile b/justfile index 72d91af..c199ea6 100755 --- a/justfile +++ b/justfile @@ -82,6 +82,10 @@ test_mypy: test_pytest: @python3 -m tox -e basic +test_autoflake: + @python3 -m autoflake --remove-all-unused-imports -r -v mangadlp/ + @python3 -m autoflake --check --remove-all-unused-imports -r -v mangadlp/ + test_tox: @python3 -m tox @@ -111,6 +115,7 @@ lint: just test_black just test_isort just test_mypy + just test_autoflake @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" tests: @@ -120,6 +125,7 @@ tests: just test_black just test_isort just test_mypy + just test_autoflake just test_pytest @echo -e "\n\033[0;32m=== ALL DONE ===\033[0m\n" diff --git a/manga-dlp.py b/manga-dlp.py index 5bc81ca..5178cd3 100644 --- a/manga-dlp.py +++ b/manga-dlp.py @@ -1,6 +1,6 @@ from mangadlp.input import main -MDLP_VERSION = "2.1.10" +MDLP_VERSION = "2.1.11" if __name__ == "__main__": main() diff --git a/mangadlp/__init__.py b/mangadlp/__init__.py index a4db992..c2a631f 100644 --- a/mangadlp/__init__.py +++ b/mangadlp/__init__.py @@ -1,22 +1,4 @@ -import logging - -from mangadlp.logger import logger_lean, logger_verbose +from mangadlp.logger import prepare_logger # prepare logger with default level INFO==20 -logging.basicConfig( - format="%(asctime)s | %(levelname)s: %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - level=20, - handlers=[logging.StreamHandler()], -) - -# create custom log levels -logging.addLevelName(15, "VERBOSE") -logging.VERBOSE = 15 # type: ignore -logging.verbose = logger_verbose # type: ignore -logging.Logger.verbose = logger_verbose # type: ignore - -logging.addLevelName(25, "LEAN") -logging.VERBOSE = 25 # type: ignore -logging.lean = logger_lean # type: ignore -logging.Logger.lean = logger_lean # type: ignore +prepare_logger() diff --git a/mangadlp/api/mangadex.py b/mangadlp/api/mangadex.py index ff0f3ae..53a7983 100644 --- a/mangadlp/api/mangadex.py +++ b/mangadlp/api/mangadex.py @@ -1,4 +1,3 @@ -import logging import re import sys from time import sleep @@ -7,6 +6,10 @@ from typing import Any import requests import mangadlp.utils as utils +from mangadlp.logger import Logger + +# prepare logger +log = Logger(__name__) class Mangadex: @@ -36,7 +39,7 @@ class Mangadex: # make initial request def get_manga_data(self) -> requests.Response: - logging.verbose(f"Getting manga data for: {self.manga_uuid}") # type: ignore + log.verbose(f"Getting manga data for: {self.manga_uuid}") counter = 1 while counter <= 3: try: @@ -45,17 +48,17 @@ class Mangadex: ) except: if counter >= 3: - logging.error("Maybe the MangaDex API is down?") + log.error("Maybe the MangaDex API is down?") sys.exit(1) else: - logging.error("Mangadex API not reachable. Retrying") + log.error("Mangadex API not reachable. Retrying") sleep(2) counter += 1 else: break # check if manga exists if manga_data.json()["result"] != "ok": - logging.error("Manga not found") + log.error("Manga not found") sys.exit(1) return manga_data @@ -68,14 +71,14 @@ class Mangadex: ) # check for new mangadex id if not uuid_regex.search(self.url_uuid): - logging.error("No valid UUID found") + log.error("No valid UUID found") sys.exit(1) manga_uuid = uuid_regex.search(self.url_uuid)[0] return manga_uuid # get the title of the manga (and fix the filename) def get_manga_title(self) -> str: - logging.verbose(f"Getting manga title for: {self.manga_uuid}") # type: ignore + log.verbose(f"Getting manga title for: {self.manga_uuid}") manga_data = self.manga_data.json() try: title = manga_data["data"]["attributes"]["title"][self.language] @@ -87,13 +90,13 @@ class Mangadex: alt_titles.update(title) title = alt_titles[self.language] except: # no title on requested language found - logging.error("Chapter in requested language not found.") + log.error("Chapter in requested language not found.") sys.exit(1) return utils.fix_name(title) # check if chapters are available in requested language def check_chapter_lang(self) -> int: - logging.verbose( # type: ignore + log.verbose( f"Checking for chapters in specified language for: {self.manga_uuid}" ) r = requests.get( @@ -102,20 +105,20 @@ class Mangadex: try: total_chapters = r.json()["total"] except: - logging.error( + log.error( "Error retrieving the chapters list. Did you specify a valid language code?" ) return 0 else: if total_chapters == 0: - logging.error("No chapters available to download!") + log.error("No chapters available to download!") return 0 return total_chapters # get chapter data like name, uuid etc def get_chapter_data(self) -> dict: - logging.verbose(f"Getting chapter data for: {self.manga_uuid}") # type: ignore + log.verbose(f"Getting chapter data for: {self.manga_uuid}") api_sorting = "order[chapter]=asc&order[volume]=asc" # check for chapters in specified lang total_chapters = self.check_chapter_lang() @@ -173,7 +176,7 @@ class Mangadex: # get images for the chapter (mangadex@home) def get_chapter_images(self, chapter: str, wait_time: float) -> list: - logging.verbose(f"Getting chapter images for: {self.manga_uuid}") # type: ignore + log.verbose(f"Getting chapter images for: {self.manga_uuid}") athome_url = f"{self.api_base_url}/at-home/server" chapter_uuid = self.manga_chapter_data[chapter][0] @@ -185,11 +188,11 @@ class Mangadex: r = requests.get(f"{athome_url}/{chapter_uuid}") api_data = r.json() if api_data["result"] != "ok": - logging.error(f"No chapter with the id {chapter_uuid} found") + log.error(f"No chapter with the id {chapter_uuid} found") api_error = True raise IndexError elif api_data["chapter"]["data"] is None: - logging.error(f"No chapter data found for chapter {chapter_uuid}") + log.error(f"No chapter data found for chapter {chapter_uuid}") api_error = True raise IndexError else: @@ -198,7 +201,7 @@ class Mangadex: except: if counter >= 3: api_error = True - logging.error(f"Retrying in a few seconds") + log.error(f"Retrying in a few seconds") counter += 1 sleep(wait_time + 2) # check if result is ok @@ -219,7 +222,7 @@ class Mangadex: # create list of chapters def create_chapter_list(self) -> list: - logging.verbose(f"Creating chapter list for: {self.manga_uuid}") # type: ignore + log.verbose(f"Creating chapter list for: {self.manga_uuid}") chapter_list = [] for chapter in self.manga_chapter_data.items(): chapter_info = self.get_chapter_infos(chapter[0]) @@ -234,9 +237,7 @@ class Mangadex: # create easy to access chapter infos def get_chapter_infos(self, chapter: str) -> dict: - logging.debug( - f"Getting chapter infos for: {self.manga_chapter_data[chapter][0]}" - ) + log.debug(f"Getting chapter infos for: {self.manga_chapter_data[chapter][0]}") chapter_uuid = self.manga_chapter_data[chapter][0] chapter_vol = self.manga_chapter_data[chapter][1] chapter_num = self.manga_chapter_data[chapter][2] diff --git a/mangadlp/app.py b/mangadlp/app.py index a58ce56..4559908 100644 --- a/mangadlp/app.py +++ b/mangadlp/app.py @@ -1,4 +1,3 @@ -import logging import re import shutil import sys @@ -7,9 +6,11 @@ from typing import Any import mangadlp.downloader as downloader import mangadlp.utils as utils - -# supported api's from mangadlp.api.mangadex import Mangadex +from mangadlp.logger import Logger + +# prepare logger +log = Logger(__name__) class MangaDLP: @@ -24,7 +25,6 @@ class MangaDLP: :param forcevol: Force naming of volumes. Useful for mangas where chapters reset each volume :param download_path: Download path. Defaults to '/downloads' :param download_wait: Time to wait for each picture to download in seconds - :param verbosity: Verbosity of the output. Uses the logging library values :return: Nothing. Just the files """ @@ -39,7 +39,6 @@ class MangaDLP: forcevol: bool = False, download_path: str = "downloads", download_wait: float = 0.5, - verbosity: int = 20, ) -> None: # init parameters self.url_uuid = url_uuid @@ -50,7 +49,6 @@ class MangaDLP: self.forcevol = forcevol self.download_path = download_path self.download_wait = download_wait - self.verbosity = verbosity # prepare everything self._prepare() @@ -74,25 +72,25 @@ class MangaDLP: # prechecks userinput/options # no url and no readin list given if not self.url_uuid: - logging.error( + log.error( 'You need to specify a manga url/uuid with "-u" or a list with "--read"' ) sys.exit(1) # checks if --list is not used if not self.list_chapters: - if self.chapters is None: + if not self.chapters: # no chapters to download were given - logging.error( + log.error( 'You need to specify one or more chapters to download. To see all chapters use "--list"' ) sys.exit(1) # if forcevol is used, but didn't specify a volume in the chapters selected if self.forcevol and ":" not in self.chapters: - logging.error("You need to specify the volume if you use --forcevol") + log.error("You need to specify the volume if you use --forcevol") sys.exit(1) # if forcevol is not used, but a volume is specified if not self.forcevol and ":" in self.chapters: - logging.error("Don't specify the volume without --forcevol") + log.error("Don't specify the volume without --forcevol") sys.exit(1) # check the api which needs to be used @@ -107,15 +105,14 @@ class MangaDLP: # check url for match if api_mangadex.search(url_uuid) or api_mangadex2.search(url_uuid): return Mangadex - # this is only for testing multiple apis - if api_test.search(url_uuid): - logging.critical("Not supported yet") + elif api_test.search(url_uuid): + log.critical("Not supported yet") sys.exit(1) # no supported api found - logging.error(f"No supported api in link/uuid found: {url_uuid}") - raise ValueError + log.error(f"No supported api in link/uuid found: {url_uuid}") + sys.exit(1) # once called per manga def get_manga(self) -> None: @@ -125,15 +122,15 @@ class MangaDLP: print_divider = "=========================================" # show infos - logging.info(f"{print_divider}") - logging.lean(f"Manga Name: {self.manga_title}") # type: ignore - logging.info(f"Manga UUID: {self.manga_uuid}") - logging.info(f"Total chapters: {len(self.manga_chapter_list)}") + log.info(f"{print_divider}") + log.lean(f"Manga Name: {self.manga_title}") + log.info(f"Manga UUID: {self.manga_uuid}") + log.info(f"Total chapters: {len(self.manga_chapter_list)}") # list chapters if list_chapters is true if self.list_chapters: - logging.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}") - logging.info(f"{print_divider}\n") + log.info(f"Available Chapters: {', '.join(self.manga_chapter_list)}") + log.info(f"{print_divider}\n") return None # check chapters to download if not all @@ -145,8 +142,8 @@ class MangaDLP: ) # show chapters to download - logging.lean(f"Chapters selected: {', '.join(chapters_to_download)}") # type: ignore - logging.info(f"{print_divider}") + log.lean(f"Chapters selected: {', '.join(chapters_to_download)}") + log.info(f"{print_divider}") # create manga folder self.manga_path.mkdir(parents=True, exist_ok=True) @@ -166,21 +163,21 @@ class MangaDLP: # chapter was not skipped except KeyError: # done with chapter - logging.info("Done with chapter\n") + log.info(f"Done with chapter '{chapter}'\n") # done with manga - logging.info(f"{print_divider}") - logging.lean(f"Done with manga: {self.manga_title}") # type: ignore + log.info(f"{print_divider}") + log.lean(f"Done with manga: {self.manga_title}") # filter skipped list skipped_chapters = list(filter(None, skipped_chapters)) if len(skipped_chapters) >= 1: - logging.lean(f"Skipped chapters: {', '.join(skipped_chapters)}") # type: ignore + log.lean(f"Skipped chapters: {', '.join(skipped_chapters)}") # filter error list error_chapters = list(filter(None, error_chapters)) if len(error_chapters) >= 1: - logging.lean(f"Chapters with errors: {', '.join(error_chapters)}") # type: ignore + log.lean(f"Chapters with errors: {', '.join(error_chapters)}") - logging.info(f"{print_divider}\n") + log.info(f"{print_divider}\n") # once called per chapter def get_chapter(self, chapter: str) -> dict: @@ -193,12 +190,12 @@ class MangaDLP: chapter, self.download_wait ) except KeyboardInterrupt: - logging.critical("Stopping") + log.critical("Stopping") sys.exit(1) # check if the image urls are empty. if yes skip this chapter (for mass downloads) if not chapter_image_urls: - logging.error( + log.error( f"No images: Skipping Vol. {chapter_infos['volume']} Ch.{chapter_infos['chapter']}" ) # add to skipped chapters list @@ -224,8 +221,7 @@ class MangaDLP: # check if chapter already exists # check for folder, if file format is an empty string if chapter_archive_path.exists(): - if self.verbosity != "lean": - logging.warning(f"'{chapter_archive_path}' already exists. Skipping") + log.warning(f"'{chapter_archive_path}' already exists. Skipping") # add to skipped chapters list return ( { @@ -240,13 +236,13 @@ class MangaDLP: chapter_path.mkdir(parents=True, exist_ok=True) # verbose log - logging.verbose(f"Chapter UUID: {chapter_infos['uuid']}") # type: ignore - logging.verbose(f"Filename: '{chapter_archive_path.name}'") # type: ignore - logging.verbose(f"File path: '{chapter_archive_path}'") # type: ignore - logging.verbose(f"Image URLS:\n{chapter_image_urls}") # type: ignore + log.verbose(f"Chapter UUID: {chapter_infos['uuid']}") + log.verbose(f"Filename: '{chapter_archive_path.name}'") + log.verbose(f"File path: '{chapter_archive_path}'") + log.verbose(f"Image URLS:\n{chapter_image_urls}") # log - logging.lean(f"Downloading: '{chapter_filename}'") # type: ignore + log.lean(f"Downloading: '{chapter_filename}'") # download images try: @@ -254,10 +250,10 @@ class MangaDLP: chapter_image_urls, chapter_path, self.download_wait ) except KeyboardInterrupt: - logging.critical("Stopping") + log.critical("Stopping") sys.exit(1) except: - logging.error(f"Cant download: '{chapter_filename}'. Skipping") + log.error(f"Cant download: '{chapter_filename}'. Skipping") # add to skipped chapters list return ( { @@ -270,23 +266,23 @@ class MangaDLP: else: # Done with chapter - logging.lean(f"INFO: Successfully downloaded: '{chapter_filename}'") # type: ignore + log.lean(f"Successfully downloaded: '{chapter_filename}'") return {"chapter_path": chapter_path} # create an archive of the chapter if needed def archive_chapter(self, chapter_path: Path) -> dict: - logging.lean(f"INFO: Creating '{self.file_format}' archive") # type: ignore + log.lean(f"Creating archive '{chapter_path}{self.file_format}'") try: # check if image folder is existing if not chapter_path.exists(): - logging.error(f"Image folder: {chapter_path} does not exist") + log.error(f"Image folder: {chapter_path} does not exist") raise IOError if self.file_format == ".pdf": utils.make_pdf(chapter_path) else: utils.make_archive(chapter_path, self.file_format) except: - logging.error(f"Archive error. Skipping chapter") + log.error(f"Archive error. Skipping chapter") # add to skipped chapters list return { "error": chapter_path, diff --git a/mangadlp/downloader.py b/mangadlp/downloader.py index d24f056..d633212 100644 --- a/mangadlp/downloader.py +++ b/mangadlp/downloader.py @@ -8,7 +8,10 @@ from typing import Union import requests import mangadlp.utils as utils +from mangadlp.logger import Logger +# prepare logger +log = Logger(__name__) # download images def download_chapter( @@ -25,21 +28,21 @@ def download_chapter( # show progress bar for default log level if logging.root.level == logging.INFO: utils.progress_bar(image_num, total_img) - logging.verbose(f"Downloading image {image_num}/{total_img}") # type: ignore + log.verbose(f"Downloading image {image_num}/{total_img}") counter = 1 while counter <= 3: try: r = requests.get(image, stream=True) if r.status_code != 200: - logging.error(f"Request for image {image} failed, retrying") + log.error(f"Request for image {image} failed, retrying") raise ConnectionError except KeyboardInterrupt: - logging.critical("Stopping") + log.critical("Stopping") sys.exit(1) except: if counter >= 3: - logging.error("Maybe the MangaDex Servers are down?") + log.error("Maybe the MangaDex Servers are down?") raise ConnectionError sleep(download_wait) counter += 1 @@ -52,7 +55,7 @@ def download_chapter( r.raw.decode_content = True shutil.copyfileobj(r.raw, file) except: - logging.error("Can't write file") + log.error("Can't write file") raise IOError image_num += 1 diff --git a/mangadlp/input.py b/mangadlp/input.py index 8e8762d..174b202 100644 --- a/mangadlp/input.py +++ b/mangadlp/input.py @@ -4,11 +4,17 @@ from pathlib import Path import mangadlp.app as app import mangadlp.logger as logger +from mangadlp.logger import Logger -MDLP_VERSION = "2.1.10" +# prepare logger +log = Logger(__name__) + +MDLP_VERSION = "2.1.11" def check_args(args): + # set logger formatting + logger.format_logger(args.verbosity) # check if --version was used if args.version: print(f"manga-dlp version: {MDLP_VERSION}") @@ -28,18 +34,21 @@ def check_args(args): # read in the list of links from a file def readin_list(readlist: str) -> list: list_file = Path(readlist) + log.verbose(f"Reading in list '{str(list_file)}'") try: url_str = list_file.read_text() url_list = url_str.splitlines() except: raise IOError - return url_list + # filter empty lines and remove them + filtered_list = list(filter(len, url_list)) + log.verbose(f"Mangas from list: {filtered_list}") + + return filtered_list def call_app(args): - # set logger formatting - logger.format_logger(args.verbosity) # call main function with all input arguments mdlp = app.MangaDLP( args.url_uuid, @@ -50,7 +59,6 @@ def call_app(args): args.forcevol, args.path, args.wait, - args.verbosity, ) mdlp.get_manga() @@ -90,6 +98,7 @@ def get_input(): # start script again with the arguments sys.argv.extend(args) + log.info(f"Args: {sys.argv}") get_args() diff --git a/mangadlp/logger.py b/mangadlp/logger.py index 817355c..c5a542b 100644 --- a/mangadlp/logger.py +++ b/mangadlp/logger.py @@ -1,6 +1,18 @@ import logging +# prepare custom levels and default config of logger +def prepare_logger(): + logging.basicConfig( + format="%(asctime)s | [%(levelname)s][%(name)s]: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=20, + handlers=[logging.StreamHandler()], + ) + logging.addLevelName(level=15, levelName="VERBOSE") + logging.addLevelName(level=25, levelName="LEAN") + + # set log message format def format_logger(verbosity: int): logging.getLogger().setLevel(verbosity) @@ -14,19 +26,37 @@ def format_logger(verbosity: int): ) else: logging.basicConfig( - format="%(asctime)s | %(levelname)s: %(message)s", + format="%(asctime)s | [%(levelname)s][%(name)s]: %(message)s", datefmt="%Y-%m-%d %H:%M:%S", force=True, ) -# create verbose logger with level 15 -def logger_verbose(msg, *args, **kwargs): - if logging.getLogger().isEnabledFor(15): - logging.log(15, msg) +class Logger: + def __init__(self, name: str): + self.name = name + # create logger + self.log = logging.getLogger(self.name) + # custom log levels + def verbose(self, message: str): + self.log.log(level=15, msg=message) -# create lean logger with level 25 -def logger_lean(msg, *args, **kwargs): - if logging.getLogger().isEnabledFor(25): - logging.log(25, msg) + def lean(self, message: str): + self.log.log(level=25, msg=message) + + # default log levels + def critical(self, message: str): + self.log.critical(msg=message) + + def error(self, message: str): + self.log.error(msg=message) + + def warning(self, message: str): + self.log.warning(msg=message) + + def info(self, message: str): + self.log.info(msg=message) + + def debug(self, message: str): + self.log.debug(msg=message) diff --git a/mangadlp/utils.py b/mangadlp/utils.py index fde1056..05b8125 100644 --- a/mangadlp/utils.py +++ b/mangadlp/utils.py @@ -1,10 +1,13 @@ -import logging import re from datetime import datetime from pathlib import Path from typing import Any from zipfile import ZipFile +from mangadlp.logger import Logger + +# prepare logger +log = Logger(__name__) # create an archive of the chapter images def make_archive(chapter_path: Path, file_format: str) -> None: @@ -24,7 +27,7 @@ def make_pdf(chapter_path: Path) -> None: try: import img2pdf except: - logging.error("Cant import img2pdf. Please install it first") + log.error("Cant import img2pdf. Please install it first") raise ImportError pdf_path = Path(f"{chapter_path}.pdf") @@ -34,7 +37,7 @@ def make_pdf(chapter_path: Path) -> None: try: pdf_path.write_bytes(img2pdf.convert(images)) except: - logging.error("Can't create '.pdf' archive") + log.error("Can't create '.pdf' archive") raise IOError diff --git a/pyproject.toml b/pyproject.toml index 75f2dd6..f5bb10e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -version = "2.1.10" +version = "2.1.11" name = "manga-dlp" description = "A cli manga downloader" readme = "README.md" diff --git a/tests/test_01_app.py b/tests/test_01_app.py index a68c9bc..0115d5f 100644 --- a/tests/test_01_app.py +++ b/tests/test_01_app.py @@ -13,6 +13,7 @@ def test_check_api_mangadex(): def test_check_api_none(): url = "https://abc.defghjk/title/abc/def" - with pytest.raises(ValueError) as e: + with pytest.raises(SystemExit) as e: app.MangaDLP(url_uuid=url, list_chapters=True, download_wait=2) - assert e.type == ValueError + assert e.type == SystemExit + assert e.value.code == 1 diff --git a/tests/test_02_utils.py b/tests/test_02_utils.py index 4c12d23..ee8ed9c 100644 --- a/tests/test_02_utils.py +++ b/tests/test_02_utils.py @@ -57,7 +57,6 @@ def test_chapter_list_full(): forcevol=True, download_path="tests", download_wait=2, - verbosity=10, ) chap_list = utils.get_chapter_list("1:1,1:2,1:4-1:7,2:", mdlp.manga_chapter_list) assert chap_list == [ diff --git a/tests/test_21_full.py b/tests/test_21_full.py index 2ba743f..f7aacf6 100644 --- a/tests/test_21_full.py +++ b/tests/test_21_full.py @@ -33,7 +33,6 @@ def test_full_api_mangadex(wait_20s): forcevol=False, download_path="tests", download_wait=2, - verbosity=10, ) mdlp.get_manga()